随州网站推广哪家专业,网上交易系统,怎么样做深网的网站,欧美做瞹瞹嗳免费视频网站我的需求是在处理几百上千万数据时避免缓存穿透以及缓存击穿情况出现#xff0c;并且确保数据库和redis同步#xff0c;为了方便我查询数据操作加了一些条件精准查询和模糊查询以及全字段模糊查询、分页、排序一些小玩意#xff0c;redis存储是hash表key值也就是数据ID…我的需求是在处理几百上千万数据时避免缓存穿透以及缓存击穿情况出现并且确保数据库和redis同步为了方便我查询数据操作加了一些条件精准查询和模糊查询以及全字段模糊查询、分页、排序一些小玩意redis存储是hash表key值也就是数据IDname值是数据表名和redis同步的别问为什么我懒
使用示例
params : utils.QueryParams{Name: users,Limit: 10,Order: id,Sort: 1,Where: map[string]interface{}{name: 张, // 将进行模糊查询age: 18, // 将进行精确匹配*: 李, // 将进行全字段模糊查询},
}
results, err : utils.CustomRedisQuery(db, redisClient, params)
if err ! nil {// 处理错误
}result, err : GetRedisById(rdb, users, 1)result, err : GetRedisByWhere(rdb, users, map[string]interface{}{status: 1, type: vip}, 1)err DeleteRedisById(rdb, users, 1)err UpdateRedisById(db, rdb, users, 1)
完整代码
/*
--------------------------------------------------------------------------------
| If this code works, it was written by Xven. If not, I dont know who wrote it.
--------------------------------------------------------------------------------
| Statement: An Ordinary Person
--------------------------------------------------------------------------------
| Author: Xven QQ:270988107
--------------------------------------------------------------------------------
| Copyright (c) 2024 Xven All rights reserved.
--------------------------------------------------------------------------------
*/
package utilsimport (contextencoding/jsonfmtsortstringssynctimegithub.com/go-redis/redis/v8gorm.io/gorm
)type QueryParams struct {Name string // 表名Limit int // 分页数量Order string // 排序字段Sort int // 排序方式 1:升序 2:降序Where map[string]interface{} // 查询条件
}/*** 检查字符串是否包含子串* Author Xven 270988107qq.com* param {string} str* param {string} substr* return {bool}*/
func containsString(str, substr string) bool {return strings.Contains(strings.ToLower(str), strings.ToLower(substr))
}/*** 检查字符串是否包含通配符* Author Xven 270988107qq.com* param {string} str* return {bool}*/
func hasWildcard(str string) bool {return strings.Contains(str, *)
}/*** 对数据进行排序* Author Xven 270988107qq.com* param {[]map[string]interface{}} data* param {string} orderField* param {int} sortType* return {void}*/
func sortData(data []map[string]interface{}, orderField string, sortType int) {sort.Slice(data, func(i, j int) bool {if sortType 1 { // 升序return fmt.Sprint(data[i][orderField]) fmt.Sprint(data[j][orderField])}return fmt.Sprint(data[i][orderField]) fmt.Sprint(data[j][orderField])})
}/*** 自定义redis查询* Author Xven 270988107qq.com* param {*gorm.DB} db* param {*redis.Client} rdb* param {QueryParams} params* return {[]map[string]interface{}, error}*/
func CustomRedisQuery(db *gorm.DB, rdb *redis.Client, params QueryParams) ([]map[string]interface{}, error) {ctx : context.Background()var result []map[string]interface{}// 参数校验,防止缓存穿透if params.Name {return nil, fmt.Errorf(表名不能为空)}// 构建 Redis keyredisKey : params.Name :list// 使用分布式锁防止缓存击穿lockKey : fmt.Sprintf(lock:%s, redisKey)lock : rdb.SetNX(ctx, lockKey, 1, 10*time.Second)if !lock.Val() {// 等待100ms后重试time.Sleep(100 * time.Millisecond)return CustomRedisQuery(db, rdb, params)}defer rdb.Del(ctx, lockKey)// 1. 先查询 Redis 缓存vals, err : rdb.HGetAll(ctx, redisKey).Result()if err nil len(vals) 0 {// 将缓存数据解析为结果集for _, v : range vals {var item map[string]interface{}if err : json.Unmarshal([]byte(v), item); err nil {result append(result, item)}}// 如果有查询条件,则进行过滤if len(params.Where) 0 {result filterData(result, params.Where)}// 处理排序if params.Order ! {sortData(result, params.Order, params.Sort)}// 处理分页if params.Limit 0 len(result) params.Limit {result result[:params.Limit]}return result, nil}// 2. Redis 没有数据,从数据库查询var dbResult []map[string]interface{}// 使用连接池控制并发pool : make(chan struct{}, 10)var wg sync.WaitGroupvar mu sync.Mutex// 使用游标分批查询数据库,避免一次性加载过多数据err db.Table(params.Name).FindInBatches(dbResult, 1000, func(tx *gorm.DB, batch int) error {wg.Add(1)pool - struct{}{} // 获取连接go func(data []map[string]interface{}) {defer func() {-pool // 释放连接wg.Done()}()pipe : rdb.Pipeline()// 批量写入Redisfor _, item : range data {// 将每条记录序列化为JSONjsonData, err : json.Marshal(item)if err ! nil {continue}// 使用ID作为field,JSON作为value写入hashid : fmt.Sprint(item[id])pipe.HSet(ctx, redisKey, id, string(jsonData))}// 执行管道命令_, err : pipe.Exec(ctx)if err ! nil {// 写入失败时重试写入数据for _, item : range data {jsonData, _ : json.Marshal(item)id : fmt.Sprint(item[id])rdb.HSet(ctx, redisKey, id, string(jsonData))}}mu.Lock()result append(result, data...)mu.Unlock()}(dbResult)return nil}).Errorwg.Wait()if err ! nil {// 设置空值缓存,防止缓存穿透rdb.Set(ctx, redisKey_empty, 1, 5*time.Minute)return nil, err}// 处理排序if params.Order ! {sortData(result, params.Order, params.Sort)}// 处理分页if params.Limit 0 len(result) params.Limit {result result[:params.Limit]}return result, nil
}/*** 过滤数据* Author Xven 270988107qq.com* param {[]map[string]interface{}} data* param {map[string]interface{}} where* return {[]map[string]interface{}}*/
func filterData(data []map[string]interface{}, where map[string]interface{}) []map[string]interface{} {var filteredResult []map[string]interface{}// 先处理精确匹配条件hasExactMatch : falsefor field, value : range where {if field ! * {if strValue, ok : value.(string); ok !hasWildcard(strValue) {hasExactMatch truebreak} else if !ok {hasExactMatch truebreak}}}if hasExactMatch {filteredResult exactMatch(data, where)if len(filteredResult) 0 {filteredResult fuzzyMatch(filteredResult, where)}} else {filteredResult fuzzyMatch(data, where)}return filteredResult
}/*** 精确匹配* Author Xven 270988107qq.com* param {[]map[string]interface{}} data* param {map[string]interface{}} where* return {[]map[string]interface{}}*/
func exactMatch(data []map[string]interface{}, where map[string]interface{}) []map[string]interface{} {var result []map[string]interface{}for _, item : range data {matched : truefor field, value : range where {if field * {continue}if strValue, ok : value.(string); ok {if !hasWildcard(strValue) {if itemValue, exists : item[field]; !exists || itemValue ! value {matched falsebreak}}} else {if itemValue, exists : item[field]; !exists || itemValue ! value {matched falsebreak}}}if matched {result append(result, item)}}return result
}/*** 模糊匹配* Author Xven 270988107qq.com* param {[]map[string]interface{}} data* param {map[string]interface{}} where* return {[]map[string]interface{}}*/
func fuzzyMatch(data []map[string]interface{}, where map[string]interface{}) []map[string]interface{} {var result []map[string]interface{}// 处理指定字段的模糊查询for _, item : range data {matched : truefor field, value : range where {if field * {continue}if strValue, ok : value.(string); ok hasWildcard(strValue) {if itemValue, exists : item[field]; exists {if strItemValue, ok : itemValue.(string); ok {pattern : strings.ReplaceAll(strValue, *, )if !strings.Contains(strings.ToLower(strItemValue), strings.ToLower(pattern)) {matched falsebreak}}}}}if matched {result append(result, item)}}// 处理全字段模糊查询if wildcardValue, exists : where[*]; exists {var globalResult []map[string]interface{}searchData : resultif len(searchData) 0 {searchData data}if strValue, ok : wildcardValue.(string); ok {for _, item : range searchData {matched : falsefor _, fieldValue : range item {if strFieldValue, ok : fieldValue.(string); ok {if containsString(strFieldValue, strValue) {matched truebreak}}}if matched {globalResult append(globalResult, item)}}}result globalResult}return result
}/*** 根据ID查询单条数据* Author Xven 270988107qq.com* param {*redis.Client} rdb* param {string} name* param {interface{}} id* return {map[string]interface{}, error}*/
func GetRedisById(rdb *redis.Client, name string, id interface{}) (map[string]interface{}, error) {ctx : context.Background()redisKey : name :list// 从Redis查询jsonData, err : rdb.HGet(ctx, redisKey, fmt.Sprint(id)).Result()if err nil {// Redis命中,解析JSON数据var result map[string]interface{}err json.Unmarshal([]byte(jsonData), result)if err nil {return result, nil}}return nil, err
}/*** 根据条件查询数据* Author Xven 270988107qq.com* param {*redis.Client} rdb* param {string} name* param {map[string]interface{}} where* param {int} is* return {interface{}, error}*/
func GetRedisByWhere(rdb *redis.Client, name string, where map[string]interface{}, is int) (interface{}, error) {ctx : context.Background()redisKey : name :list// 获取所有数据values, err : rdb.HGetAll(ctx, redisKey).Result()if err ! nil {return nil, err}var results []map[string]interface{}// 遍历所有数据进行条件匹配for _, jsonStr : range values {var item map[string]interface{}err : json.Unmarshal([]byte(jsonStr), item)if err ! nil {continue}// 检查是否匹配所有条件match : truefor k, v : range where {if item[k] ! v {match falsebreak}}if match {results append(results, item)// 如果是单条查询且已找到,则直接返回if is 0 {return item, nil}}}if is 0 {return nil, nil}return results, nil
}/*** 删除指定ID的数据* Author Xven 270988107qq.com* param {*redis.Client} rdb* param {string} name* param {interface{}} id* return {error}*/
func DeleteRedisById(rdb *redis.Client, name string, id interface{}) error {ctx : context.Background()redisKey : name :listmaxRetries : 3for i : 0; i maxRetries; i {err : rdb.HDel(ctx, redisKey, fmt.Sprint(id)).Err()if err nil {return nil}// 重试前等待短暂时间time.Sleep(time.Millisecond * 100)}return fmt.Errorf(failed to delete after %d retries, maxRetries)
}/*** 更新指定ID的数据* Author Xven 270988107qq.com* param {*gorm.DB} db* param {*redis.Client} rdb* param {string} name* param {interface{}} id* return {error}*/
func UpdateRedisById(db *gorm.DB, rdb *redis.Client, name string, id interface{}) error {ctx : context.Background()redisKey : name :listmaxRetries : 3// 从数据库查询数据var result map[string]interface{}err : db.Table(name).Where(id ?, id).Take(result).Errorif err ! nil {return err}// 序列化数据jsonData, err : json.Marshal(result)if err ! nil {return err}// 重试更新Redisfor i : 0; i maxRetries; i {err rdb.HSet(ctx, redisKey, fmt.Sprint(id), string(jsonData)).Err()if err nil {return nil}time.Sleep(time.Millisecond * 100)}return fmt.Errorf(failed to update after %d retries, maxRetries)
}
