Skip to content

coderdream_20220506

Jump to bottom
CoderDream edited this page May 6, 2022 · 1 revision 
package com.coderdream.utils;

/**
 * @author ：CoderDream
 * @date ：Created in 2022/4/28 20:10
 * @description：
 * @modified By：CoderDream
 * @version: $
 */
public class Constants {
    /**
     * 62个字符
     */
    public static final String CHARS = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
    /**
     * 布隆过滤器的容量 100000000
     */
    public static final Long BLOOM_FILTER_INSERTION = 100000000L;

    /**
     * 期望的误判率
     */
    public static final double BLOOM_FILTER_FPP = 0.01;

    /**
     * Guava Cache 并发级别为8，并发级别是指可以同时写缓存的线程数
     */
    public static final Integer CONCURRENCY_LEVEL = 8;

    /**
     * Guava Cache 缓存容器的初始容量为 10
     */
    public static final Integer INITIAL_CAPACITY = 10;

    /**
     * Guava Cache 设置缓存最大容量为 100000000 Long 的最大值 9,223,372,036,854,775,807 6 位 62 进制数的可能的组合 56,800,235,584
     */
    public static final Long MAXIMUM_SIZE = 100000000L;

    /**
     * Guava Cache 设置缓存的水位
     */
    public static final Float WATER_LEVEL = 0.95F;


}
package com.coderdream.generator;

public interface CodeGenerator {
    /**
     * 产生的code
     *
     * @return
     */
    String generateCode();

}












package com.coderdream.helper;

import com.coderdream.utils.CommonUtil;
import com.coderdream.utils.Constants;

import org.springframework.stereotype.Component;

/**
 * @author ：CoderDream
 * @date ：Created in 2022/4/25 20:12
 * @description：
 * @modified By：CoderDream
 * @version: $
 */
@Component
public class ShortLinkHelper {
    /**
     * 生成短链码
     *
     * @param param
     * @return
     */
    public String createShortLinkCode(String param) {
        if(param == null || "".equals(param.trim())) {
            return "";
        }
        long murmurhash = CommonUtil.murmurHash32(param);
        //进制转换
        return encodeToBase62(murmurhash);
    }

    /**
     * 10进制转62进制
     *
     * @param num
     * @return
     */
    private String encodeToBase62(long num) {
        // StringBuffer线程安全，StringBuilder线程不安全
        StringBuffer sb = new StringBuffer();
        do {
            int i = (int) (num % 62);
            sb.append(Constants.CHARS.charAt(i));
            num = num / 62;
        } while (num > 0);
        return sb.reverse().toString();
    }
}



package com.coderdream.helper;

import com.coderdream.utils.Config;
import com.coderdream.utils.Constants;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;

import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.util.concurrent.TimeUnit;

/**
 * @author ：CoderDream
 * @date ：Created in 2022/4/26 20:08
 * @description：
 * @modified By：CoderDream
 * @version: $
 */
@Service
@Slf4j
public class GuavaCacheHelper {
    @Resource
    private Config config;

    private Cache<Object, Object> cache;

    @PostConstruct
    public void init() {
        cache = CacheBuilder.newBuilder()
                // 设置并发级别为8，并发级别是指可以同时写缓存的线程数，此处最好和cpu数一致（Runtime.getRuntime().availableProcessors()）
                .concurrencyLevel(Constants.CONCURRENCY_LEVEL)
                // 设置缓存容器的初始容量为10
                .initialCapacity(Constants.INITIAL_CAPACITY)
                // 设置缓存最大容量为100，超过100之后就会按照LRU最近最少使用算法来移除缓存项，生产环境可设置为 100000000
                .maximumSize(Constants.MAXIMUM_SIZE)
                // 是否需要统计缓存情况，该操作消耗一定的性能，生产环境应该去除
                .recordStats()
                // 设置写缓存后n秒钟过期
                .expireAfterWrite(config.EXPIRE_SEC, TimeUnit.SECONDS)
                // 设置读写缓存后n秒钟过期，实际很少用到，类似于expireAfterWrite
                //.expireAfterAccess(17, TimeUnit.SECONDS)
                // 只阻塞当前数据加载线程，其他线程返回旧值
                //.refreshAfterWrite(13, TimeUnit.SECONDS)
                // 设置缓存的移除通知
                .removalListener(notification -> {
                    log.error(notification.getKey() + " " + notification.getValue() + " 被移除,原因:" + notification.getCause());
                })
                //build方法中可以指定CacheLoader，在缓存不存在时通过CacheLoader的实现自动加载缓存
                .build();
    }

    /**
     * 获取缓存
     */
    public Object get(String key) {
        return StringUtils.isEmpty(key) ? null : cache.getIfPresent(key);
    }


    /**
     * 获取缓存大小
     */
    public long size() {
        return cache.size();
    }

    /**
     * 放入缓存
     */
    public void put(String key, Object value) {
        if (StringUtils.isNotEmpty(key) && value != null) {
            cache.put(key, value);
        } else {
            return;
        }
    }
}


package com.coderdream.generator;

import com.coderdream.service.LinkService;
import com.coderdream.utils.Constants;
import com.google.common.hash.BloomFilter;
import com.google.common.hash.Funnels;

import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import javax.annotation.Resource;

@RunWith(SpringRunner.class)
@SpringBootTest
public class StringGeneratorTest {
    //
    // @Resource
    // private StringGenerator stringGenerator;

    @Test
    public void testGetShortLink() {
        StringGenerator stringGenerator = new StringGenerator();
        for (int i = 0; i < 64; i++) {
            System.out.println(stringGenerator.generateCode());
        }
        // String shortLink = linkService.getShortLink("http://www.baidu.com/jLFYW7ZlML-344410");
        // Assert.assertNotNull(shortLink);
        // Assert.assertEquals("http://www.baidu.com/jLFYW7ZlML-344410", linkService.getLongLink(shortLink));
    }

}


package com.coderdream.helper;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;

import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;

import javax.annotation.Resource;

@RunWith(SpringRunner.class)
@SpringBootTest
public class ShortLinkHelperTest {

    @Resource
    private ShortLinkHelper shortLinkHelper;

    @Test
    public void testCreateShortLinkCode() {
        assertEquals("",shortLinkHelper.createShortLinkCode(null));
        assertEquals("",shortLinkHelper.createShortLinkCode(""));
        assertEquals("1BWWGy", shortLinkHelper.createShortLinkCode("http://www.baidu.com/jLFYW7ZlML-344410"));
    }
}


package com.coderdream.service.impl;

import com.coderdream.bean.ShortLinkBean;
import com.coderdream.generator.RandomStringGenerator;
import com.coderdream.generator.StringGenerator;
import com.coderdream.helper.BloomFilterHelper;
import com.coderdream.helper.FileOperateHelper;
import com.coderdream.helper.GeneratorHelper;
import com.coderdream.helper.GuavaCacheHelper;
import com.coderdream.service.LinkService;
import com.coderdream.utils.Config;
import com.coderdream.utils.Constants;
import com.coderdream.utils.DuplicatedEnum;
import com.coderdream.helper.ShortLinkHelper;

import lombok.extern.slf4j.Slf4j;

import org.springframework.stereotype.Service;

import java.math.BigInteger;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import javax.annotation.PostConstruct;
import javax.annotation.Resource;

@Service
@Slf4j
public class LinkServiceImpl implements LinkService {
    @Resource
    private Config config;

    @Resource
    private FileOperateHelper fleOperateHelper;

    @Resource
    private ShortLinkHelper shortLinkHelper;

    @Resource
    private GuavaCacheHelper guavaCacheHelper;

    @Resource
    private BloomFilterHelper bloomFilterHelper;

    @Resource
    private GeneratorHelper generatorHelper;

    /**
     * 号码生成器列表
     */
    @Resource
    private RandomStringGenerator randomStringGenerator;

    private String generateCode;


    private Set<String> hashCodeSet;

    @PostConstruct
    public void init() {
        //stringGeneratorList = generatorHelper.createStringGenerator();
        generateCode = randomStringGenerator.generateCode();
        hashCodeSet = new HashSet<>();
        //  machineId = Long.parseLong(fileOperateAbility.readFile("machineId"));
    }

    @Override
    public String getShortLink(String longLink) {
        // 校验longLink
        if (longLink == null || "".equals(longLink)) {
            // 记录日志
            log.error("入参错误，不能为空：" + longLink);
            return "";
        }
        // 获取机器ID，这里写在文件中，后期通过ZooKeeper管理
        String machineId = fleOperateHelper.readFile("machineId");
        // 生成短链接
        String code = shortLinkHelper.createShortLinkCode(longLink);

        // 记录日志
        //log.error("缓存大小：" + guavaCacheHelper.size());
        Long guavaCacheSize = guavaCacheHelper.size();
        Long initSize = Constants.MAXIMUM_SIZE;
        Float rate = Float.valueOf(guavaCacheSize) / Float.valueOf(initSize);
        // 记录日志
       // log.error("目前缓存水位：" + rate);
       //  if (Constants.WATER_LEVEL < rate) {
            generateCode = randomStringGenerator.generateCode();
        // }
        // 机器ID作为前缀
        code = machineId + generateCode + code;

        // 创建短链接Bean
        ShortLinkBean shortLinkBean = new ShortLinkBean();
        shortLinkBean.setShortLink(code);
        shortLinkBean.setLongLink(longLink);
        shortLinkBean.setExpireTime(System.currentTimeMillis() + config.EXPIRE_SEC * 1000);
        // 如果存在（可能误判）
        if (bloomFilterHelper.mightContain(code)) {
            // 从缓存中取对象
            ShortLinkBean oldShortLinkBean = (ShortLinkBean) guavaCacheHelper.get(code);
            // 如果不存在误判为存在，则直接将新的数据写入缓存中
            if (oldShortLinkBean == null) {
                // 把短链接放入Guava缓存中
                guavaCacheHelper.put(code, shortLinkBean);
                // 把短链接放入布隆过滤器
                bloomFilterHelper.put(code);
                // 记录日志
                log.warn("布隆过滤器误判： new code: " + code + " ; new link: " + longLink);
            }
            // 如果确实存在
            else {
                String oldLongLink = oldShortLinkBean.getLongLink();
                // 判断是否Hash冲突了(code相同，长链接url不同)
                if (code.equals(oldShortLinkBean.getShortLink()) && !longLink.equals(oldLongLink)) {
                    String totalCode = code;
                    // 记录日志
                    log.warn("Hash冲突, old and new code: " + code + "; old link: " + oldLongLink + " ; new link: "
                        + longLink);
                    // 构造新code、新link
                    // code加上枚举前缀后再取Hash，生成新的短链接
                    code = machineId + generateCode + shortLinkHelper.createShortLinkCode(
                        DuplicatedEnum.DUPLICATED.getKey() + "_" + code);

                    totalCode = totalCode + "_" + code;
                    log.error("TotalCode：" + totalCode);
                    hashCodeSet.add(totalCode);
                    // 长链接加上前缀
                    String newLongLink = DuplicatedEnum.DUPLICATED.getKey() + "_" + longLink;
                    log.error("Hash冲突解决： new code: " + code + "; old link: " + oldShortLinkBean.getLongLink()
                        + " ; new link: " + newLongLink);
                    // 设置新的短链接
                    shortLinkBean.setShortLink(code);
                    // 设置新的长链接
                    shortLinkBean.setLongLink(newLongLink);
                    // 把短链接放入Guava缓存中
                    guavaCacheHelper.put(code, shortLinkBean);
                    // 把短链接放入布隆过滤器
                    bloomFilterHelper.put(code);
                }
                // 未冲突，已存在数据，不做处理，既不放到缓存中，也不放到过滤器中
                else {
                    // 记录日志
                    log.info("已存在： code " + code + " ; longLink: " + longLink);
                }
            }
        }
        // 通过布隆过滤器判断：如果不存在（100%正确），则直接放入缓存中
        else {
            // 把短链接放入Guava缓存中
            guavaCacheHelper.put(code, shortLinkBean);
            // 把短链接放入布隆过滤器
            bloomFilterHelper.put(code);
        }
        // 记录日志
       // log.info("Hash冲突集合大小： " + hashCodeSet.size());
        // 将短链接返回给调用方
        return code;
    }

    @Override
    public String getLongLink(String shortLink) {
        // 校验
        if (shortLink == null || "".equals(shortLink)) {
            // 记录日志
            log.error("入参错误，不能为空：" + shortLink);
            return "";
        }
        // 从缓存中获取对象
        ShortLinkBean shortLinkBean = (ShortLinkBean) guavaCacheHelper.get(shortLink);
        // 如果不存在，则记日志，然后返回空
        if (shortLinkBean == null) {
            log.warn("缓存中不存在 " + shortLink + " 对应的长链接");
            return "";
        }
        // 如果存在，处理长链接
        else {
            // 取出长链接
            String longLink = shortLinkBean.getLongLink();
            // 如果不存在Hash冲突标记，则直接返回长链接
            if (!longLink.startsWith(DuplicatedEnum.DUPLICATED.getKey())) {
                return longLink;
            }
            // 否则去掉冲突标记后再返回
            else {
                log.warn("去掉冲突标记后再返回：" + longLink);
                longLink = longLink.replace(DuplicatedEnum.DUPLICATED.getKey() + "_", "");
            }
            // 将长链接返回给调用方
            return longLink;
        }
    }
}


package com.coderdream.helper;

import com.coderdream.generator.StringGenerator;
import com.coderdream.utils.Constants;

import lombok.extern.slf4j.Slf4j;

import org.springframework.stereotype.Service;

import java.util.ArrayList;
import java.util.List;

/**
 * @author XY
 * @Description
 * @createTime 2021年12月17日 21:21:00
 */
@Service
@Slf4j
public class GeneratorHelper {

    /**
     * 获取队列发号器
     *
     */
    public List<StringGenerator> createStringGenerator() {
        List<StringGenerator> list = new ArrayList<>();
        int n = Constants.CHARS.toCharArray().length;
        for (Long i = 0L; i < n; i++) {
            list.add(new StringGenerator());
            log.info("[createStringGenerator],队列发号器创建,id={}", i);
        }
        return list;
    }
}


package com.coderdream.generator;

import com.coderdream.utils.Constants;

import lombok.extern.slf4j.Slf4j;

import org.springframework.stereotype.Service;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;

/**
 * 功能描述
 *
 * @since 2022-05-05
 */
@Slf4j
@Service
public class StringGenerator implements CodeGenerator {

    private ArrayBlockingQueue<String> queue;

    @Override
    public String generateCode() {
        return queue.poll();
    }

    public StringGenerator() {

        Integer queueSizeConfig = Constants.CHARS.toCharArray().length;
        queue = new ArrayBlockingQueue<>(queueSizeConfig);
        List<String> charList = new ArrayList<>();
        for (int i = 0; i < queueSizeConfig; i++) {
            Character character = Constants.CHARS.toCharArray()[i];
            charList.add(character.toString());
        }

        //后台线程
        Thread thread = new Thread(() -> {
            int index = 0;
            while (index < queueSizeConfig) {
                // if (this.low >= this.highMax) {
                //     log.error("已用尽号码,停止后台线程,id={},low={},high={},step={},highMax={}", id, low, high, step, highMax);
                //     return;
                // }
                // //如果低水位达到高水位,高水位上移
                // if (low >= high) {
                //     high += step;
                //     //高水位不可越过最高水位
                //     if (high >= highMax) {
                //         high = highMax;
                //     }
                //     //todo 持久化记录high
                // }
                // try {
                //     //采用阻塞方式放入元素
                //     queue.put(charList.get(index));
                // } catch (InterruptedException e) {
                //     log.info("[CacheQueueNumberGenerator] daemon thread interrupted", e);
                // }
                //低水位移动
                index++;
                //todo 可以加提前预警，比如low>highMax*0.8
            }
        });
        thread.setDaemon(true);
        thread.start();

        int index = 0;
        while (index < queueSizeConfig) {
            // if (this.low >= this.highMax) {
            //     log.error("已用尽号码,停止后台线程,id={},low={},high={},step={},highMax={}", id, low, high, step, highMax);
            //     return;
            // }
            // //如果低水位达到高水位,高水位上移
            // if (low >= high) {
            //     high += step;
            //     //高水位不可越过最高水位
            //     if (high >= highMax) {
            //         high = highMax;
            //     }
            //     //todo 持久化记录high
            // }
            try {
                //采用阻塞方式放入元素
                queue.put(charList.get(index));
            } catch (InterruptedException e) {
                log.info("[CacheQueueNumberGenerator] daemon thread interrupted", e);
            }
            //低水位移动
            index++;
            //todo 可以加提前预警，比如low>highMax*0.8
        }

        System.out.println(queue.size());
    }

}



package com.coderdream.generator;

import com.coderdream.utils.Constants;

import lombok.extern.slf4j.Slf4j;

import org.springframework.stereotype.Service;

import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.concurrent.ArrayBlockingQueue;

/**
 * 功能描述
 *
 * @since 2022-05-05
 */
@Slf4j
@Service
public class RandomStringGenerator {

    private List<String> stringList = new ArrayList<>();

    public String generateCode() {
        Random random = new Random();
        int index = random.nextInt(Constants.CHARS.toCharArray().length);
        return stringList.get(index) ;
    }

    public RandomStringGenerator() {

        Integer queueSizeConfig = Constants.CHARS.toCharArray().length;
        for (int i = 0; i < queueSizeConfig; i++) {
            Character character = Constants.CHARS.toCharArray()[i];
            stringList.add(character.toString());
        }

        System.out.println(stringList.size());
    }

}



              


          



  
          

            


            
Clone this wiki locally