欢迎您访问程序员文章站本站旨在为大家提供分享程序员计算机编程知识!
您现在的位置是: 首页

Semaphore控制高并发下载导致内存溢出问题

程序员文章站 2022-07-12 20:10:16
...

        在项目实际应用中,由于下载文件内容都比较大,如果同时有很多用户同时在下载,JVM的内存就会升的很高,甚至崩溃。为了避免很多用户同时下载,特引入Semaphore控制一次最多有配置个线程能进入实际下载的代码,即而控制JVM内存不会升的很高而导致崩溃。

import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.Semaphore;

import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.io.IOUtils;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.stereotype.Component;

@Component("DownloadView.csv")
public class DownloadView extends AbstractCsvView implements InitializingBean {
	//允许的最大线程数
	private String threadNum=PropertyUtil.getProperty("threadNum");
	// 线程池
    private ExecutorService exec = null;
    // 只能threadNum个线程同时访问
    private Semaphore semp = new Semaphore(Integer.parseInt(threadNum), true);
    
    @Override
    protected void buildExcelDocument(Map<String, Object> model, List<String> csvList, HttpServletRequest request,
            HttpServletResponse response) throws Exception  {
    	String fileName ="";
        String url = request.getParameter("outputInfo");
        if(StringUtil.isNotEmpty(url)){
        	fileName= url.substring(url.lastIndexOf("/") + 1, url.length());
        }
        super.setUrl(url);
        try {
			response.setHeader("Content-Disposition",
			        "attachment;filename=" + URLEncoder.encode(fileName, response.getCharacterEncoding()));
		} catch (UnsupportedEncodingException e) {
			throw new Exception("不支持此编码格式");
		}
    }
    
    @SuppressWarnings("unchecked")
    @Override
    protected void renderMergedOutputModel(final Map<String, Object> model, HttpServletRequest request,
            HttpServletResponse response) throws Exception, IOException, InterruptedException, ExecutionException  {
    	
    	exec = Executors.newCachedThreadPool();
    	response.setContentType(getContentType());
    	final String url = request.getParameter("outputInfo");
    	final ServletOutputStream out=response.getOutputStream();
    	final HttpServletRequest _request = request;
    	final HttpServletResponse _response = response;
    	InputStream in=null;
    	try{
    		in= new FileInputStream(url);
    	}catch(Exception e){
    		throw new Exception("找不到对应的文件:"+url);
    	}
    	final InputStream fis=in;
    	final String encode=super.getEncoding();
    	Callable<Boolean> call = new Callable<Boolean>() {
        	@Override
            public Boolean call() {
                try {
                    // 获取许可
                    semp.acquire();
                    List<String> csvList = null;
                    //IOUtils.readLines()是一次性读取整个文件
                    //readline() 和 .readlines()之间的差异是后者一次读取整个文件,像read()一样。
                    //readlines()自动将文件内容分析成一个行的列表,
                    //readline()每次只读取一行,通常比 readlines()慢得多。
                    //仅当没有足够内存可以一次读取整个文件时,才应该使用readline().
            		csvList = IOUtils.readLines(fis);
                    buildExcelDocument(model, csvList, _request, _response);
                    if (encode == null) {
            			IOUtils.writeLines(csvList,encode, out);
                    } else {
            			IOUtils.writeLines(csvList, encode, out, encode);
                    }
                    //Thread.sleep((long) (2000));
                    return true;
                }  catch (Exception e) {
                	System.out.println(e);
					return false;
				} finally {
					semp.release();
				}
            }
        };
        Future<Boolean> future=null;
        if(!exec.isShutdown()){
        	future= exec.submit(call);
        }
       exec.shutdown();
	   if((Boolean) future.get()){
		   System.out.println("success");
	   }else{
		   System.out.print("fail");
	   }
    }
    
    @Override
    public void afterPropertiesSet() throws Exception {
    }
}

AbstractCsvView.java

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Map;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.io.IOUtils;
import org.springframework.core.io.Resource;
import org.springframework.core.io.support.LocalizedResourceHelper;
import org.springframework.web.servlet.support.RequestContextUtils;
import org.springframework.web.servlet.view.AbstractView;

public abstract class AbstractCsvView  extends AbstractView{

	/** The content type for an csv response */
    private static final String CONTENT_TYPE = "text/csv";
    
    /** The extension to look for existing templates */
    private static final String EXTENSION = ".csv";
    
    private String lineEnding;
    
    private String encoding;
    
    /** The url at which the template to use is located */
    private String url;
    
    /**
     * Default Constructor.
     * Sets the content type of the view to "text/csv".
     */
    public AbstractCsvView() {
            setContentType(CONTENT_TYPE);
    }
    
    /**
     * Set the URL of the Excel workbook source, without localization part nor extension.
     */
    public void setUrl(String url) {
            this.url = url;
    }
    
    
    public void setLineEnding(String lineEnding) {
            this.lineEnding = lineEnding;
    }

    public void setEncoding(String encoding) {
            this.encoding = encoding;
    }
    
    

    public String getLineEnding() {
        return lineEnding;
    }

    public String getEncoding() {
        return encoding;
    }

    public String getUrl() {
        return url;
    }

    @Override
    protected boolean generatesDownloadContent() {
            return true;
    }
    
    @Override
    protected void renderMergedOutputModel(Map<String, Object> model,
                    HttpServletRequest request, HttpServletResponse response)
                    throws Exception {
            // Set the content type and get the output stream.
            response.setContentType(getContentType());
            List<String> csvList = null;
            if (this.url != null) {
                    InputStream in = getTemplateSource(this.url, request);
                    csvList = IOUtils.readLines(in);
            }else{
                    csvList = new ArrayList<String>();
            }
            buildExcelDocument(model, csvList, request, response);
            if(this.encoding == null){
                    IOUtils.writeLines(csvList, this.lineEnding, response.getOutputStream());
            }else{
                    IOUtils.writeLines(csvList, this.lineEnding, response.getOutputStream(), this.encoding);
            }
    }

    protected InputStream getTemplateSource(String url, HttpServletRequest request) throws IOException {
            LocalizedResourceHelper helper = new LocalizedResourceHelper(getApplicationContext());
            Locale userLocale = RequestContextUtils.getLocale(request);
            Resource inputFile = helper.findLocalizedResource(url, EXTENSION, userLocale);
            
            // Create the Excel document from the source.
            if (logger.isDebugEnabled()) {
                    logger.debug("Loading Excel workbook from " + inputFile);
            }
            return inputFile.getInputStream();
    }
    
    /**
     * Subclasses must implement this method to create an csv List
     * document, given the model.
     * @param model the model Map
     * @param csvList
     * @param request in case we need locale etc. Shouldn't look at attributes.
     * @param response in case we need to set cookies. Shouldn't write to it.
     * @throws Exception in case of failure
     */
    protected abstract void buildExcelDocument(Map<String, Object> model, List<String> csvList,
                    HttpServletRequest request, HttpServletResponse response) throws Exception;

}