diff --git a/.gitignore b/.gitignore index 9154f4c..e0bb4ad 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,5 @@ hs_err_pid* replay_pid* +.idea +target diff --git a/db/a.html b/db/a.html new file mode 100644 index 0000000..e69de29 diff --git a/db/init-database.sql b/db/init-database.sql new file mode 100644 index 0000000..c36593f --- /dev/null +++ b/db/init-database.sql @@ -0,0 +1,16 @@ +create table madou_video_info +( + id bigserial primary key not null, + title varchar(255) not null, + classify varchar(255) not null, + cover_url varchar(255), + m3u8_url varchar(255) +); +create table classify_info +( + id bigserial primary key not null, + name varchar(255) not null, + url varchar(255) not null, + page_size int not null +); +select sum(page_size) * 18 / 1024 / 2 from classify_info; \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..6f0ca8c --- /dev/null +++ b/pom.xml @@ -0,0 +1,54 @@ + + + 4.0.0 + crawler-video + top.yuchat + 0.0.1 + crawler-video + jar + 某个网站的视频爬取 + + + org.springframework.boot + spring-boot-parent + 3.2.3 + + + + + + org.projectlombok + lombok + provided + + + + org.apache.commons + commons-lang3 + + + + org.springframework.boot + spring-boot-starter-web + + + + com.baomidou + mybatis-plus-spring-boot3-starter + 3.5.7 + + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + \ No newline at end of file diff --git a/src/main/java/top/yuchat/crawler/video/CrawlerVideoApplication.java b/src/main/java/top/yuchat/crawler/video/CrawlerVideoApplication.java new file mode 100644 index 0000000..f327962 --- /dev/null +++ b/src/main/java/top/yuchat/crawler/video/CrawlerVideoApplication.java @@ -0,0 +1,12 @@ +package top.yuchat.crawler.video; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class CrawlerVideoApplication { + + public static void main(String[] args) { + SpringApplication.run(CrawlerVideoApplication.class, args); + } +} diff --git a/src/main/java/top/yuchat/crawler/video/config/MybatisConfig.java b/src/main/java/top/yuchat/crawler/video/config/MybatisConfig.java new file mode 100644 index 0000000..e1561aa --- /dev/null +++ b/src/main/java/top/yuchat/crawler/video/config/MybatisConfig.java @@ -0,0 +1,37 @@ +package top.yuchat.crawler.video.config; + +import com.baomidou.mybatisplus.core.handlers.MetaObjectHandler; +import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor; +import com.baomidou.mybatisplus.extension.plugins.inner.OptimisticLockerInnerInterceptor; +import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor; +import org.apache.ibatis.reflection.MetaObject; +import org.mybatis.spring.annotation.MapperScan; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.time.LocalDateTime; + +@Configuration +@MapperScan("top.yuchat.crawler.video.**.mapper") +public class MybatisConfig implements MetaObjectHandler { + + @Bean + public MybatisPlusInterceptor mybatisPlusInterceptor(){ + MybatisPlusInterceptor mpi = new MybatisPlusInterceptor(); + mpi.addInnerInterceptor(new PaginationInnerInterceptor()); + mpi.addInnerInterceptor(new OptimisticLockerInnerInterceptor()); + return mpi; + } + + @Override + public void insertFill(MetaObject metaObject) { + this.strictInsertFill(metaObject, "createTime", LocalDateTime.class, LocalDateTime.now()); + this.strictInsertFill(metaObject, "updateTime", LocalDateTime.class, LocalDateTime.now()); + } + + @Override + public void updateFill(MetaObject metaObject) { + this.strictUpdateFill(metaObject, "updateTime", LocalDateTime.class, LocalDateTime.now()); + } + +} diff --git a/src/main/java/top/yuchat/crawler/video/models/entity/MadouVideoInfo.java b/src/main/java/top/yuchat/crawler/video/models/entity/MadouVideoInfo.java new file mode 100644 index 0000000..cdae7f8 --- /dev/null +++ b/src/main/java/top/yuchat/crawler/video/models/entity/MadouVideoInfo.java @@ -0,0 +1,21 @@ +package top.yuchat.crawler.video.models.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import lombok.Data; + +@Data +public class MadouVideoInfo { + + @TableId(type = IdType.AUTO) + private Long id; + + private String title; + + private String classify; + + private String coverUrl; + + private String m3u8Url; + +} diff --git a/src/main/java/top/yuchat/crawler/video/models/mapper/MadouVideoMapper.java b/src/main/java/top/yuchat/crawler/video/models/mapper/MadouVideoMapper.java new file mode 100644 index 0000000..0c54c0a --- /dev/null +++ b/src/main/java/top/yuchat/crawler/video/models/mapper/MadouVideoMapper.java @@ -0,0 +1,7 @@ +package top.yuchat.crawler.video.models.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import top.yuchat.crawler.video.models.entity.MadouVideoInfo; + +public interface MadouVideoMapper extends BaseMapper { +} diff --git a/src/main/java/top/yuchat/crawler/video/models/service/MadouVideoService.java b/src/main/java/top/yuchat/crawler/video/models/service/MadouVideoService.java new file mode 100644 index 0000000..1197913 --- /dev/null +++ b/src/main/java/top/yuchat/crawler/video/models/service/MadouVideoService.java @@ -0,0 +1,21 @@ +package top.yuchat.crawler.video.models.service; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import org.springframework.stereotype.Service; +import top.yuchat.crawler.video.models.entity.MadouVideoInfo; +import top.yuchat.crawler.video.models.mapper.MadouVideoMapper; + +@Service +public class MadouVideoService extends ServiceImpl { + + /** + * /index.php/vod/type/id/3.html + * 1-10 + * 20-27 + * 29-30 + * + * /index.php/vod/type/id/1/page/2.html + * + * https://yutujx.com/?url=https://t20a.cdn2020.com/video/m3u8/2022/10/23/cc234c9c/index.m3u8 + */ +}