wangys/schoolNews
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

敏感词

Browse Source
This commit is contained in:
wangys
2025-11-21 18:56:14 +08:00
parent 028e748565
commit c2cac51762
23 changed files with 108710 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
schoolNewsServ/.bin/mysql/sql/createTableSensitive.sql Normal file
View File

@@ -0,0 +1,10 @@
use school_news;
DROP TABLE IF EXISTS `tb_sensitive_word`;
CREATE TABLE `tb_sensitive_word` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`word` varchar(255) NOT NULL COMMENT '敏感词',
`type` VARCHAR(10) NOT NULL COMMENT '类型allow\deny'
PRIMARY KEY (`id`),
UNIQUE KEY `word` (`word`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='敏感词表';

2
schoolNewsServ/.bin/mysql/sql/initAll.sql
View File

@@ -41,6 +41,8 @@ SOURCE createTableCrontab.sql;
SOURCE createTableMessage.sql;
SOURCE createTableSensitive.sql;
-- =====================================================
-- 插入初始数据
-- =====================================================

22
schoolNewsServ/api/api-sensitive/pom.xml Normal file
View File

@@ -0,0 +1,22 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.xyzh</groupId>
<artifactId>api</artifactId>
<version>1.0.0</version>
</parent>
<groupId>org.xyzh</groupId>
<artifactId>api-sensitive</artifactId>
<version>1.0.0</version>
<packaging>jar</packaging>
<properties>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
</properties>
</project>

13
schoolNewsServ/api/api-sensitive/src/main/java/org/xyzh/api/sensitive/SensitiveService.java Normal file
View File

@@ -0,0 +1,13 @@
package org.xyzh.api.sensitive;
import org.xyzh.common.core.domain.ResultDomain;
import org.xyzh.common.dto.sensitive.TbSensitive;
public interface SensitiveService {
ResultDomain<Boolean> addSensitiveWord(TbSensitive sensitive);
ResultDomain<Boolean> deleteSensitiveWord(TbSensitive sensitive);
ResultDomain<Boolean> judgeSensitive(String text);
}

6
schoolNewsServ/api/pom.xml
View File

@@ -25,6 +25,7 @@
<module>api-file</module>
<module>api-crontab</module>
<module>api-message</module>
<module>api-sensitive</module>
</modules>
<properties>
@@ -94,6 +95,11 @@
<artifactId>api-message</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>org.xyzh</groupId>
<artifactId>api-sensitive</artifactId>
<version>1.0.0</version>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>

39
schoolNewsServ/common/common-dto/src/main/java/org/xyzh/common/dto/sensitive/TbSensitive.java Normal file
View File

@@ -0,0 +1,39 @@
package org.xyzh.common.dto.sensitive;
import java.io.Serializable;
public class TbSensitive implements Serializable{
private static final long serialVersionUID = 1L;
private Integer id;
private String word;
private String type;
public Integer getId() {
return id;
}
public void setId(Integer id) {
this.id = id;
}
public String getWord() {
return word;
}
public void setWord(String word) {
this.word = word;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
}

6
schoolNewsServ/pom.xml
View File

@@ -22,6 +22,7 @@
<module>file</module>
<module>crontab</module>
<module>message</module>
<module>sensitive</module>
</modules>
<properties>
@@ -90,6 +91,11 @@
<artifactId>system</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>org.xyzh</groupId>
<artifactId>sensitive</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>org.xyzh</groupId>
<artifactId>news</artifactId>

84
schoolNewsServ/sensitive/pom.xml Normal file
View File

@@ -0,0 +1,84 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.xyzh</groupId>
<artifactId>school-news</artifactId>
<version>1.0.0</version>
</parent>
<groupId>org.xyzh</groupId>
<artifactId>sensitive</artifactId>
<version>1.0.0</version>
<packaging>jar</packaging>
<properties>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
<sensitive-word.version>0.24.0</sensitive-word.version>
</properties>
<dependencies>
<dependency>
<groupId>org.xyzh</groupId>
<artifactId>api-sensitive</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>com.github.houbb</groupId>
<artifactId>sensitive-word</artifactId>
<version>${sensitive-word.version}</version>
<exclusions>
<exclusion>
<groupId>com.github.houbb</groupId>
<artifactId>sensitive-word-data</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- Common模块依赖 -->
<dependency>
<groupId>org.xyzh</groupId>
<artifactId>common-all</artifactId>
<version>1.0.0</version>
</dependency>
<!-- Spring Boot Web -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- MyBatis Plus for Spring Boot 3 -->
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-spring-boot3-starter</artifactId>
</dependency>
<!-- MySQL驱动 -->
<dependency>
<groupId>com.mysql</groupId>
<artifactId>mysql-connector-j</artifactId>
</dependency>
<!-- 数据库连接池 -->
<dependency>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
</dependency>
<!-- Log4j2 -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-log4j2</artifactId>
</dependency>
</dependencies>
</project>

45
schoolNewsServ/sensitive/src/main/java/org/xyzh/sensitive/config/SensitiveWordInitializer.java Normal file
View File

@@ -0,0 +1,45 @@
package org.xyzh.sensitive.config;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;
import org.xyzh.api.sensitive.SensitiveService;
import org.xyzh.common.redis.service.RedisService;
import org.xyzh.sensitive.constants.SensitiveRedisContants;
import org.xyzh.sensitive.mapper.SensitiveMapper;
import org.xyzh.common.dto.sensitive.TbSensitive;
@Component
public class SensitiveWordInitializer implements CommandLineRunner{
private static final Logger logger = LoggerFactory.getLogger(SensitiveWordInitializer.class);
@Autowired
private RedisService redisService;
@Autowired
private SensitiveMapper sensitiveMapper;
@Override
public void run(String... args) throws Exception {
logger.info("敏感词初始化");
// 数据库读取吸入redis
List<TbSensitive> list = sensitiveMapper.selectAll();
Set<String> allowSet = new HashSet<>();
Set<String> denySet = new HashSet<>();
list.stream().forEach(item->{
if (item.getType().equals("allow")) {
allowSet.add(item.getWord());
}else {
denySet.add(item.getWord());
}
});
redisService.sAdd(SensitiveRedisContants.SENSITIVE_WORD_ALLOW, allowSet);
redisService.sAdd(SensitiveRedisContants.SENSITIVE_WORD_DENY, denySet);
}
}

36
schoolNewsServ/sensitive/src/main/java/org/xyzh/sensitive/config/SpringSensitiveWordConfig.java Normal file
View File

@@ -0,0 +1,36 @@
package org.xyzh.sensitive.config;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.xyzh.sensitive.word.DynamicAllowWord;
import org.xyzh.sensitive.word.DynamicDenyWord;
import com.github.houbb.sensitive.word.bs.SensitiveWordBs;
import com.github.houbb.sensitive.word.support.allow.WordAllows;
@Configuration
public class SpringSensitiveWordConfig {
@Autowired
private DynamicAllowWord dynamicAllowWord;
@Autowired
private DynamicDenyWord dynamicDenyWord;
/**
* 初始化引导类
* @return 初始化引导类
* @since 1.0.0
*/
@Bean
public SensitiveWordBs sensitiveWordBs() {
SensitiveWordBs sensitiveWordBs = SensitiveWordBs.newInstance()
.wordAllow(WordAllows.chains(WordAllows.defaults(), dynamicAllowWord))
.wordDeny(dynamicDenyWord)
// 各种其他配置
.init();
return sensitiveWordBs;
}
}

6
schoolNewsServ/sensitive/src/main/java/org/xyzh/sensitive/constants/SensitiveRedisContants.java Normal file
View File

@@ -0,0 +1,6 @@
package org.xyzh.sensitive.constants;
public class SensitiveRedisContants {
public static final String SENSITIVE_WORD_ALLOW = "sensitive:word:allow";
public static final String SENSITIVE_WORD_DENY = "sensitive:word:deny";
}

18
schoolNewsServ/sensitive/src/main/java/org/xyzh/sensitive/mapper/SensitiveMapper.java Normal file
View File

@@ -0,0 +1,18 @@
package org.xyzh.sensitive.mapper;
import java.util.List;
import org.apache.ibatis.annotations.Mapper;
import org.xyzh.common.dto.sensitive.TbSensitive;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
@Mapper
public interface SensitiveMapper extends BaseMapper<TbSensitive> {
public List<TbSensitive> selectAll();
public Integer addSensitiveWord(TbSensitive tbSensitive);
public Integer deleteSensitiveWord(TbSensitive tbSensitive);
}

107
schoolNewsServ/sensitive/src/main/java/org/xyzh/sensitive/service/SensitiveServiceImpl.java Normal file
View File

@@ -0,0 +1,107 @@
package org.xyzh.sensitive.service;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.xyzh.api.sensitive.SensitiveService;
import org.xyzh.common.core.domain.ResultDomain;
import org.xyzh.common.dto.sensitive.TbSensitive;
import org.xyzh.common.redis.service.RedisService;
import org.xyzh.sensitive.constants.SensitiveRedisContants;
import org.xyzh.sensitive.mapper.SensitiveMapper;
import java.util.List;
import com.github.houbb.sensitive.word.core.SensitiveWordHelper;
@Service
public class SensitiveServiceImpl implements SensitiveService{
@Autowired
private SensitiveMapper sensitiveMapper;
@Autowired
private SensitiveWordHelper sensitiveWordHelper;
@Autowired
private RedisService redisService;
@Override
public ResultDomain<Boolean> addSensitiveWord(TbSensitive tbSensitive) {
ResultDomain<Boolean> resultDomain = new ResultDomain<>();
String word = tbSensitive.getWord();
String type = tbSensitive.getType();
// 第一次检查：Redis中是否已存在
String redisKey = "allow".equals(type) ? SensitiveRedisContants.SENSITIVE_WORD_ALLOW : SensitiveRedisContants.SENSITIVE_WORD_DENY;
if (redisService.sMembers(redisKey).contains(word)) {
resultDomain.fail("敏感词已存在");
return resultDomain;
}
// 加锁
synchronized (this) {
// 第二次检查：再次确认Redis中是否已存在
if (redisService.sMembers(redisKey).contains(word)) {
resultDomain.fail("敏感词已存在");
return resultDomain;
}
// 执行数据库插入
int i = sensitiveMapper.addSensitiveWord(tbSensitive);
if (i > 0) {
// 同步到Redis
redisService.sAdd(redisKey, word);
resultDomain.success("添加敏感词成功", true);
} else {
resultDomain.fail("添加敏感词失败");
}
}
return resultDomain;
}
@Override
public ResultDomain<Boolean> deleteSensitiveWord(TbSensitive tbSensitive) {
ResultDomain<Boolean> resultDomain = new ResultDomain<>();
String word = tbSensitive.getWord();
String type = tbSensitive.getType();
// 第一次检查：Redis中是否存在
String redisKey = "allow".equals(type) ? SensitiveRedisContants.SENSITIVE_WORD_ALLOW : SensitiveRedisContants.SENSITIVE_WORD_DENY;
if (!redisService.sMembers(redisKey).contains(word)) {
resultDomain.fail("敏感词不存在");
return resultDomain;
}
// 加锁
synchronized (this) {
// 第二次检查：再次确认Redis中是否存在
if (!redisService.sMembers(redisKey).contains(word)) {
resultDomain.fail("敏感词不存在");
return resultDomain;
}
// 执行数据库删除
int i = sensitiveMapper.deleteSensitiveWord(tbSensitive);
if (i > 0) {
// 从Redis中移除
redisService.sRemove(redisKey, word);
resultDomain.success("删除敏感词成功", true);
} else {
resultDomain.fail("删除敏感词失败");
}
}
return resultDomain;
}
@Override
public ResultDomain<Boolean> judgeSensitive(String text) {
ResultDomain<Boolean> resultDomain = new ResultDomain<>();
boolean containsSensitive = sensitiveWordHelper.contains(text);
resultDomain.success("敏感词检测完成", containsSensitive);
return resultDomain;
}
}

35
schoolNewsServ/sensitive/src/main/java/org/xyzh/sensitive/word/DynamicAllowWord.java Normal file
View File

@@ -0,0 +1,35 @@
package org.xyzh.sensitive.word;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.xyzh.common.redis.service.RedisService;
import org.xyzh.sensitive.constants.SensitiveRedisContants;
import com.github.houbb.sensitive.word.api.IWordAllow;
@Component
public class DynamicAllowWord implements IWordAllow {
@Autowired
private RedisService redisService;
@Override
public List<String> allow() {
Set<Object> members = redisService.sMembers(SensitiveRedisContants.SENSITIVE_WORD_ALLOW);
if (members == null || members.isEmpty()) {
return Collections.emptyList();
}
List<String> list = new ArrayList<>(members.size());
for (Object o : members) {
if (o != null) {
list.add(String.valueOf(o));
}
}
return list;
}
}

35
schoolNewsServ/sensitive/src/main/java/org/xyzh/sensitive/word/DynamicDenyWord.java Normal file
View File

@@ -0,0 +1,35 @@
package org.xyzh.sensitive.word;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.xyzh.common.redis.service.RedisService;
import org.xyzh.sensitive.constants.SensitiveRedisContants;
import com.github.houbb.sensitive.word.api.IWordDeny;
@Component
public class DynamicDenyWord implements IWordDeny {
@Autowired
private RedisService redisService;
@Override
public List<String> deny() {
Set<Object> members = redisService.sMembers(SensitiveRedisContants.SENSITIVE_WORD_DENY);
if (members == null || members.isEmpty()) {
return Collections.emptyList();
}
List<String> list = new ArrayList<>(members.size());
for (Object o : members) {
if (o != null) {
list.add(String.valueOf(o));
}
}
return list;
}
}

33
schoolNewsServ/sensitive/src/main/resources/mapper/SensitiveMapper.xml Normal file
View File

@@ -0,0 +1,33 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="org.xyzh.sensitive.mapper.SensitiveMapper">
<resultMap id="BaseMap" resultType="org.xyzh.common.dto.sensitive.TbSensitive">
<id column="id" property="id"/>
<result column="word" property="word"/>
<result column="type" property="type"/>
</resultMap>
<sql id="Base_Column_List">
id, word, type
</sql>
<select id="selectAll" resultMap="BaseMap">
SELECT
<include refid="Base_Column_List"/>
FROM tb_sensitive
</select>
<insert id="addSensitiveWord" parameterType="org.xyzh.common.dto.sensitive.TbSensitive" useGeneratedKeys="true" keyProperty="id" keyColumn="id">
INSERT INTO tb_sensitive (word, type)
VALUES (#{word}, #{type})
</insert>
<delete id="deleteSensitiveWord" parameterType="org.xyzh.common.dto.sensitive.TbSensitive">
DELETE FROM tb_sensitive
WHERE word = #{word}
<if test="type != null and type != ''">
AND type = #{type}
</if>
</delete>
</mapper>

0
schoolNewsServ/sensitive/src/main/resources/sensitiveData/SensitiveMapper.xml Normal file
View File

12
schoolNewsServ/sensitive/src/main/resources/sensitiveData/sensitive_word_allow.txt Normal file
View File

@@ -0,0 +1,12 @@
生日快乐
曹操
幸运
幸运儿
17年前
1条
1梯两户
1比1
年检
幸存
恶搞
游戏机

0
schoolNewsServ/sensitive/src/main/resources/sensitiveData/sensitive_word_deny.txt Normal file
View File

64419
schoolNewsServ/sensitive/src/main/resources/sensitiveData/sensitive_word_dict.txt Normal file
View File

File diff suppressed because it is too large Load Diff

12
schoolNewsServ/sensitive/src/main/resources/sensitiveData/sensitive_word_dict_en.txt Normal file
View File

@@ -0,0 +1,12 @@
fuck
duck
shit
chicken
fowl
sex
sexy
prostitute
whore
harlot
hooker
gender

43768
schoolNewsServ/sensitive/src/main/resources/sensitiveData/sensitive_word_tags.txt Normal file
View File

File diff suppressed because it is too large Load Diff