crawler-video/src/main/java/top/yuchat/crawler/video/utils/HttpRequestComponent.java

package top.yuchat.crawler.video.utils;

import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.ssl.SSLContextBuilder;
import org.apache.http.util.EntityUtils;
import org.apache.tomcat.util.http.fileupload.IOUtils;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;

@Slf4j
@Component
public class HttpRequestComponent {
    // private static final HttpHost PROXY = new HttpHost("172.20.0.1", 1080);

    private CloseableHttpClient closeableHttpClient;
    private RequestConfig requestConfig;
    private int socketTimeout = 5000;
    private int connectTimeout = 5000;
    private int connectRequestTimeout = 5000;
    private int maxTotal = 500;
    private int maxPerRoute = 120;

    public HttpRequestComponent() throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
        init();
        createHttpClients();
    }

    public void createHttpClients() throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
        SSLContextBuilder builder = new SSLContextBuilder();
        builder.loadTrustMaterial(null, (_2, _1) -> true);
        SSLConnectionSocketFactory sslref = new SSLConnectionSocketFactory(builder.build(), NoopHostnameVerifier.INSTANCE);
        Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create().register("http", new PlainConnectionSocketFactory()).register("https", sslref).build();
        PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registry);
        cm.setMaxTotal(maxTotal);
        cm.setDefaultMaxPerRoute(maxPerRoute);
        closeableHttpClient = HttpClients.custom()
                .setSSLSocketFactory(sslref)
                .setConnectionManager(cm)
                .setRetryHandler(new DefaultHttpRequestRetryHandler(5, true))
                .setConnectionManagerShared(true)
                .build();
    }

    public void init() {
        this.requestConfig = RequestConfig.custom()
                .setSocketTimeout(socketTimeout)
                .setConnectTimeout(connectTimeout)
                .setConnectionRequestTimeout(connectRequestTimeout)
//                .setProxy(PROXY)
                .build();
    }

    public String get(String url) throws IOException {
        HttpGet httpGet = new HttpGet(url);
        httpGet.setConfig(requestConfig);
        CloseableHttpResponse response = closeableHttpClient.execute(httpGet);
        int statusCode = response.getStatusLine().getStatusCode();
        if (HttpStatus.SC_OK != statusCode) {
            log.error("请求失败，状态码：{}", statusCode);
            throw new RuntimeException("请求失败，状态码：" + statusCode);
        }
        HttpEntity entity = response.getEntity();
        return EntityUtils.toString(entity);
    }

    public void download(String url, Path path) throws IOException {
        HttpGet httpGet = new HttpGet(url);
        httpGet.setConfig(requestConfig);
        CloseableHttpResponse response = closeableHttpClient.execute(httpGet);
        int statusCode = response.getStatusLine().getStatusCode();
        if (HttpStatus.SC_OK != statusCode) {
            log.error("文件下载失败，状态码：{}", statusCode);
            throw new RuntimeException("文件下载失败");
        }
        HttpEntity entity = response.getEntity();

        if (Files.exists(path)) {
            long contentLength = entity.getContentLength();
            long size = Files.size(path);
            log.warn("文件已存在，请求文件大小：{}，本地文件大小：{}", contentLength, size);
            if (size >= contentLength) {
                log.warn("文件已存在，跳过下载，文件路径：{}", path);
                response.close();
                return;
            }
            log.warn("文件不完整，删除重新下载，PATH：{}", path);
            Files.delete(path);
        }
        Files.copy(entity.getContent(), path);
    }


    public void download(String url, OutputStream os)  {
        HttpGet httpGet = new HttpGet(url);
        httpGet.setConfig(requestConfig);
        try (CloseableHttpResponse response = closeableHttpClient.execute(httpGet)) {
            int statusCode = response.getStatusLine().getStatusCode();
            if (HttpStatus.SC_OK != statusCode) {
                log.error("文件下载失败，状态码：{}", statusCode);
                throw new RuntimeException("文件下载失败");
            }
            HttpEntity entity = response.getEntity();
            try (InputStream content = entity.getContent()) {
                IOUtils.copy(content, os);
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        } finally {
            httpGet.releaseConnection();
        }

    }

    public boolean checkUrl(String url) {
        HttpGet httpGet = new HttpGet(url);
        httpGet.setConfig(requestConfig);
        CloseableHttpResponse response = null;
        try {
            response = closeableHttpClient.execute(httpGet);
            int statusCode = response.getStatusLine().getStatusCode();
            if (HttpStatus.SC_OK == statusCode) {
                return true;
            }
        } catch (Exception ignored) {
        } finally {
            if (response != null) {
                try {
                    response.close();
                } catch (IOException e) {
                    log.error("Failed to close response", e);
                }
            }
            // 关闭 httpGet 请求
            httpGet.releaseConnection();
        }
        return false;
    }


    public long getFileSize(String url) throws IOException {
        HttpGet httpGet = new HttpGet(url);
        httpGet.setConfig(requestConfig);
        CloseableHttpResponse response = closeableHttpClient.execute(httpGet);
        int statusCode = response.getStatusLine().getStatusCode();
        if (HttpStatus.SC_OK != statusCode) {
            log.error("文件下载失败，状态码：{}", statusCode);
            throw new RuntimeException("文件下载失败");
        }
        HttpEntity entity = response.getEntity();
        long contentLength = entity.getContentLength();
        response.close();
        return contentLength;
    }
}