压缩上传到ftp服务器后拉取数据解压缩
程序员文章站
2022-05-15 12:53:07
...
压缩工具类_这里压缩专门提供了个方法返回流的方式直接提交到ftp服务器
/**
*
* @program: demo
* @description: ZipFileUtil
* @author: jinbiao
* @create: 2021-04-16 09:26
**/
@Slf4j
public class UnZipFileUtil {
/**
* 缓存区大小
*/
private static final int BUFFER_SIZE = 2 * 1024;
/**
* @param multipartFile 需解压文件
* @param target 解压后路径
* @throws IOException
*/
public static List<String> zipFile(MultipartFile multipartFile, String target) throws IOException {
return unZip(multipartFile2File(multipartFile), target);
}
/**
* @param source 需解压zip路径
* @param target 解压后zip路径
*/
public static List<String> zipString(String source, String target) throws IOException {
File sourceFile = new File(source);
File[] files = sourceFile.listFiles();
List<String> list = new ArrayList<>();
if (null == files && files.length < 1) {
return null;
}
for (File file : files) {
String suffix = file.getName().substring(file.getName().lastIndexOf(".") + 1);
if (file.length() < 1 || !suffix.equals("zip")) {
file.delete();
continue;
}
list = unZip(sourceFile, target);
}
return list;
}
private static List<String> unZip(File file, String target) throws IOException {
List<String> result = new ArrayList<>();
List<String> serviceList = new ArrayList<>();
/**
* 解压zip
*/
InputStream fileStream = null;
FileOutputStream out = null;
ZipFile zipFile = null;
try {
zipFile = new ZipFile(file, Charset.forName("gbk"));
Enumeration<? extends ZipEntry> entries = zipFile.entries();
if (null != entries) {
while (entries.hasMoreElements()) {
ZipEntry zipEntry = entries.nextElement();
fileStream = zipFile.getInputStream(zipEntry);
String name = zipEntry.getName();
String outPath = (target + name);
String substring = outPath.substring(0, outPath.lastIndexOf("/"));
File newfile = new File(substring);
if (!newfile.exists()) {
newfile.mkdirs();
}
//是否是文件夹
if (new File(outPath).isDirectory()) {
continue;
}
if (outPath.endsWith("htm") && !outPath.contains("header.htm")) {
serviceList.add(name.substring(name.lastIndexOf("/") + 1));
}
out = new FileOutputStream(outPath);
int buffer = 0;
while ((buffer = fileStream.read()) != -1) {
out.write(buffer);
}
}
}
} catch (Exception e) {
log.info("解压错误");
} finally {
fileStream.close();
out.close();
zipFile.close();
// file.delete();
}
for (String serviceName : serviceList) {
result.add(String.format("%s%s", target, serviceName));
}
return result;
}
public static File multipartFile2File(MultipartFile multipartFile) {
File file = new File(multipartFile.getOriginalFilename());
try (InputStream in = multipartFile.getInputStream();
OutputStream out = new FileOutputStream(file)) {
int byteRead;
while ((byteRead = in.read()) != -1) {
out.write(byteRead);
}
return file;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* 压缩核心方法
*/
public static void compress(ZipOutputStream zos, String path, String name, String data) throws Exception {
byte[] buf = new byte[BUFFER_SIZE];
zos.putNextEntry(new ZipEntry(StringUtils.isNotEmpty(path) ? path + name : name));
int len;
ByteArrayInputStream in = new ByteArrayInputStream(data.getBytes());
while ((len = in.read(buf)) != -1) {
zos.write(buf, 0, len);
}
zos.closeEntry();
in.close();
}
public static InputStream getCompressed(String name, String data) throws IOException {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
byte[] buf = new byte[BUFFER_SIZE];
ZipOutputStream zos = new ZipOutputStream(bos);
ByteArrayInputStream in = new ByteArrayInputStream(data.getBytes());
ZipEntry entry = new ZipEntry(name);
zos.putNextEntry(entry);
int count;
while ((count = in.read(buf)) != -1) {
zos.write(buf, 0, count);
}
in.close();
zos.closeEntry();
zos.close();
return new ByteArrayInputStream(bos.toByteArray());
}
ftp工具类
/**
* <p>Title: </p>
*
* <p>Description: </p>
*
* @author
* @QQ
* @date 2019年5月21
*/
@Component
@Slf4j
public class FTPUtils {
public static final Logger logger = LoggerFactory.getLogger(FTPUtils.class);
public static String LOCAL_CHARSET = "GBK";
private FTPClient ftpClient = null;
@Value("")
private String server;
@Value("")
private int port;
@Value("ftpuser")
private String userName;
@Value("hame")
private String userPassword;
@Value("/")
private String path;
/**
* <p>Title: 连接服务器</p>
*
* <p>Description: </p>
*
* @return 连接成功与否 true:成功， false:失败
* @author Yang
* @date 2019年5月21
*/
public boolean open() {
// 判断是否已连接
if (ftpClient != null && ftpClient.isConnected()) {
return true;
}
try {
ftpClient = new FTPClient();
// 连接FTP服务器
ftpClient.setRemoteVerificationEnabled(false);
ftpClient.connect(this.server, this.port);
// 如果采用默认端口，可以使用ftp.connect(host)的方式直接连接FTP服务器
ftpClient.login(this.userName, this.userPassword);
// 检测连接是否成功
int reply = ftpClient.getReplyCode();
// 开启服务器对UTF-8的支持，如果服务器支持就用UTF-8编码，否则就使用本地编码（GBK）.
if (FTPReply.isPositiveCompletion(ftpClient.sendCommand("OPTS UTF8", "ON"))) {
LOCAL_CHARSET = "UTF-8";
}
ftpClient.setControlEncoding(LOCAL_CHARSET);
if (!FTPReply.isPositiveCompletion(reply)) {
logger.info("FTP服务器拒绝连接.");
this.close();
return false;
// System.exit(1);
}
logger.info("FTP连接成功:" + this.server + ";port:" + this.port + ";name:" + this.userName + ";pwd:" + this.userPassword);
// ftpClient.setFileType(FTP.BINARY_FILE_TYPE); // 设置上传模式.binally or ascii
return true;
} catch (Exception e) {
this.close();
e.printStackTrace();
return false;
}
}
/**
* <p>Title: 切换到父目录</p>
*
* <p>Description: </p>
*
* @return 切换结果 true：成功， false：失败
* @author Yang
* @date 2019年5月21
*/
private boolean changeToParentDir() {
try {
return ftpClient.changeToParentDirectory();
} catch (IOException e) {
e.printStackTrace();
return false;
}
}
/**
* <p>Title: 改变当前目录到指定目录</p>
*
* <p>Description: </p>
*
* @param dir 目的目录
* @return 切换结果 true：成功，false：失败
* @author Yang
* @date 2019年5月21
*/
private boolean cd(String dir) {
try {
return ftpClient.changeWorkingDirectory(dir);
} catch (IOException e) {
e.printStackTrace();
return false;
}
}
/**
* <p>Title: 获取目录下所有的文件名称</p>
*
* <p>Description: </p>
*
* @param filePath 指定的目录
* @return 文件列表, 或者null
* @author Yang
* @date 2019年5月21
*/
public FTPFile[] getFileList(String filePath) throws IOException {
// ftpClient.doCommand("opts", "utf8 off");
ftpClient.enterLocalPassiveMode();
try {
return ftpClient.listFiles(filePath);
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
/**
* <p>Title: 层层切换工作目录</p>
*
* <p>Description: </p>
*
* @param ftpPath 目的目录
* @return 切换结果
* @author Yang
* @date 2019年5月21
*/
public boolean changeDir(String ftpPath) {
if (!ftpClient.isConnected()) {
return false;
}
try {
// 将路径中的斜杠统一
char[] chars = ftpPath.toCharArray();
StringBuffer sbStr = new StringBuffer(256);
for (int i = 0; i < chars.length; i++) {
if ('\\' == chars[i]) {
sbStr.append('/');
} else {
sbStr.append(chars[i]);
}
}
ftpPath = sbStr.toString();
if (ftpPath.indexOf('/') == -1) {
// 只有一层目录
ftpClient.changeWorkingDirectory(new String(ftpPath.getBytes(), "UTF-8"));
} else {
// 多层目录循环创建
String[] paths = ftpPath.split("/");
for (int i = 0; i < paths.length; i++) {
ftpClient.changeWorkingDirectory(new String(paths[i].getBytes(), "UTF-8"));
}
}
return true;
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
/**
* <p>Title: 循环创建目录，并且创建完目录后，设置工作目录为当前创建的目录下</p>
*
* <p>Description: </p>
*
* @param ftpPath 需要创建的目录
* @return
* @author Yang
* @date 2019年5月21
*/
public boolean mkDir(String ftpPath) {
if (!ftpClient.isConnected()) {
return false;
}
try {
// 将路径中的斜杠统一
char[] chars = ftpPath.toCharArray();
StringBuffer sbStr = new StringBuffer(256);
for (int i = 0; i < chars.length; i++) {
if ('\\' == chars[i]) {
sbStr.append('/');
} else {
sbStr.append(chars[i]);
}
}
ftpPath = sbStr.toString();
logger.info("ftpPath:" + ftpPath);
if (ftpPath.indexOf('/') == -1) {
// 只有一层目录
ftpClient.makeDirectory(new String(ftpPath.getBytes(), "UTF-8"));
ftpClient.changeWorkingDirectory(new String(ftpPath.getBytes(), "UTF-8"));
} else {
// 多层目录循环创建
String[] paths = ftpPath.split("/");
for (int i = 0; i < paths.length; i++) {
ftpClient.makeDirectory(new String(paths[i].getBytes(), "UTF-8"));
ftpClient.changeWorkingDirectory(new String(paths[i].getBytes(), "UTF-8"));
}
}
return true;
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
/**
* <p>Title: 上传文件到FTP服务器</p>
*
* <p>Description: </p>
*
* @param localDirectoryAndFileName 本地文件目录和文件名
* @param ftpFileName 上传到服务器的文件名()
* @param ftpDirectory FTP目录如:/path1/pathb2/,如果目录不存在会自动创建目录(目录可以省略)
* @return
* @author Yang
* @date 2019年5月21
*/
public boolean upload(String localDirectoryAndFileName, String ftpFileName, String ftpDirectory) {
if (!ftpClient.isConnected()) {
return false;
}
boolean flag = false;
if (ftpClient != null) {
FileInputStream fis = null;
try {
//获取文件后缀
String format = localDirectoryAndFileName.split("\\.")[1];
File srcFile = new File(localDirectoryAndFileName);
fis = new FileInputStream(srcFile);
// 创建目录
this.mkDir(path + ftpDirectory);
ftpClient.setBufferSize(100000);
ftpClient.setControlEncoding("UTF-8");
// 设置文件类型（二进制）
ftpClient.setFileType(FTPClient.BINARY_FILE_TYPE);
if (ftpFileName == null || ftpFileName == "") {
ftpFileName = srcFile.getName();
}
ftpFileName = ftpFileName + "." + format;
// 上传
flag = ftpClient.storeFile(new String((ftpDirectory + "/" + ftpFileName).getBytes(), "UTF-8"), fis);
logger.info("上传文件成功，本地文件名： " + localDirectoryAndFileName + "，上传到目录：" + ftpDirectory + "/" + ftpFileName);
} catch (Exception e) {
this.close();
e.printStackTrace();
return false;
} finally {
try {
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return flag;
}
/**
* <p>Title: 上传文件到FTP服务器</p>
*
* <p>Description: </p>
*
* @param file 本地文件目录和文件名
* @param ftpFileName 上传到服务器的文件名()
* @param ftpDirectory FTP目录如:/path1/pathb2/,如果目录不存在会自动创建目录(目录可以省略)
* @return
* @author Yang
* @date 2019年5月21
*/
public boolean uploadMultipartfile(MultipartFile file, String ftpFileName, String ftpDirectory) {
if (!ftpClient.isConnected()) {
return false;
}
boolean flag = false;
if (ftpClient != null) {
InputStream fis = null;
try {
//获取文件后缀
String format = file.getOriginalFilename().split("\\.")[1];
File srcFile = new File(file.getOriginalFilename());
fis = file.getInputStream();
// 创建目录
ftpDirectory = path + ftpDirectory;
this.mkDir(ftpDirectory);
ftpClient.setBufferSize(100000);
ftpClient.setControlEncoding("UTF-8");
// 设置文件类型（二进制）
ftpClient.setFileType(FTPClient.BINARY_FILE_TYPE);
if (ftpFileName == null || ftpFileName == "") {
ftpFileName = srcFile.getName();
}
/* ftpFileName = ftpFileName + "." + format;*/
// 上传
flag = ftpClient.storeFile(new String((ftpDirectory + "/" + ftpFileName + "." + format).getBytes(), "UTF-8"), fis);
logger.info("上传文件成功，本地文件名： " + file.getName() + "，上传到目录：" + path + ftpDirectory + "/" + ftpFileName);
} catch (Exception e) {
this.close();
e.printStackTrace();
return false;
} finally {
try {
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return flag;
}
/**
* <p>Title: 上传文件到FTP服务器</p>
*
* <p>Description: </p>
*
* @param file 本地文件目录和文件名
* @param ftpFileName 上传到服务器的文件名()
* @param ftpDirectory FTP目录如:/path1/pathb2/,如果目录不存在会自动创建目录(目录可以省略)
* @return
* @author Yang
* @date 2019年5月21
*/
public boolean uploadfile(File file, String ftpFileName, String ftpDirectory) {
if (!ftpClient.isConnected()) {
return false;
}
boolean flag = false;
ftpClient.enterLocalPassiveMode();
if (ftpClient != null) {
FileInputStream fis = null;
try {
fis = new FileInputStream(file);
// 创建目录
ftpClient.setBufferSize(100000);
ftpClient.setControlEncoding("UTF-8");
// 设置文件类型（二进制）
ftpClient.setFileType(FTPClient.BINARY_FILE_TYPE);
if (ftpFileName == null || ftpFileName == "") {
ftpFileName = file.getName();
}
// 上传
flag = ftpClient.storeFile(new String((ftpDirectory + "/" + file.getName()).getBytes(), "UTF-8"), fis);
logger.info("上传文件成功，本地文件名： " + file.getName() + "，上传到目录：" + ftpDirectory + "/" + ftpFileName);
} catch (Exception e) {
this.close();
e.printStackTrace();
return false;
} finally {
try {
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return flag;
}
/**
* 上传文件
*
* @param
* @param saveName 全路径。如/home/public/a.txt
* @param fis 要上传的文件流
* @return
*/
public boolean storeFile(String saveName, InputStream fis) {
if (!ftpClient.isConnected()) {
return false;
}
boolean flag = false;
ftpClient.enterLocalPassiveMode();
if (ftpClient != null) {
try {
// 创建目录
ftpClient.setBufferSize(100000);
// // 设置文件类型（二进制）
ftpClient.setFileType(FTPClient.BINARY_FILE_TYPE);
// 上传
flag = ftpClient.storeFile(new String(saveName.getBytes("utf-8"), "ISO-8859-1"), fis);
logger.info("上传文件成功，本地文件名： " + saveName + "，上传到目录：" + path + "/" + saveName);
} catch (Exception e) {
this.close();
e.printStackTrace();
return false;
} finally {
try {
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return flag;
}
/**
* <p>Title: 从FTP服务器上下载文件</p>
*
* <p>Description: </p>
*
* @param ftpDirectoryAndFileName ftp服务器文件路径，以/dir形式开始
* @param localDirectoryAndFileName 保存到本地的目录
* @return
* @author Yang
* @date 2019年5月21
*/
public boolean get(String ftpDirectoryAndFileName, String localDirectoryAndFileName, String original_file_name) {
if (!ftpClient.isConnected()) {
return false;
}
ftpClient.enterLocalPassiveMode(); // Use passive mode as default
try {
// 将路径中的斜杠统一
ftpDirectoryAndFileName = path + ftpDirectoryAndFileName;
char[] chars = ftpDirectoryAndFileName.toCharArray();
StringBuffer sbStr = new StringBuffer(256);
for (int i = 0; i < chars.length; i++) {
if ('\\' == chars[i]) {
sbStr.append('/');
} else {
sbStr.append(chars[i]);
}
}
ftpDirectoryAndFileName = sbStr.toString();
// String filePath = ftpDirectoryAndFileName.substring(0, ftpDirectoryAndFileName.lastIndexOf("/"));
//
// String fileName = ftpDirectoryAndFileName.substring(ftpDirectoryAndFileName.lastIndexOf("/") + 1);
//
// this.changeDir(filePath);
ftpClient.retrieveFile(new String(ftpDirectoryAndFileName.getBytes(), "UTF-8"), new FileOutputStream(localDirectoryAndFileName + "/" + original_file_name)); // download
// file
logger.info(ftpClient.getReplyString()); // check result
logger.info("从ftp服务器上下载文件：" + ftpDirectoryAndFileName + "， 保存到：" + localDirectoryAndFileName);
return true;
} catch (IOException e) {
e.printStackTrace();
return false;
}
}
/**
* <p>Title: 返回FTP目录下的文件列表</p>
*
* <p>Description: </p>
*
* @param pathName
* @return
* @author Yang
* @date 2019年5月21
*/
public String[] getFileNameList(String pathName) {
try {
return ftpClient.listNames(pathName);
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
/**
* <p>Title: 删除FTP上的文件</p>
*
* <p>Description: </p>
*
* @param ftpDirAndFileName 路径开头不能加/，比如应该是test/filename1
* @return
* @author Yang
* @date 2019年5月21
*/
public boolean deleteFile(String ftpDirAndFileName) throws IOException {
if (!ftpClient.isConnected()) {
return false;
}
if (!ftpClient.changeWorkingDirectory(path)) {
System.out.println("切换目录失败：" + path);
return false;
}
ftpClient.enterLocalPassiveMode();
try {
String name = ftpDirAndFileName.substring(0, ftpDirAndFileName.lastIndexOf("."));
// ftpDirAndFileName = path + ftpDirAndFileName;
boolean b = ftpClient.deleteFile(new String(ftpDirAndFileName.getBytes("utf-8"), "iso-8859-1"));
logger.info("删除竣工文件: " + "文件名路径" + ftpDirAndFileName);
return true;
} catch (IOException e) {
e.printStackTrace();
return false;
}
}
/**
* <p>Title: 删除FTP目录</p>
*
* <p>Description: </p>
*
* @param ftpDirectory
* @return
* @author Yang
* @date 2019年5月21
*/
public boolean deleteDirectory(String ftpDirectory) {
if (!ftpClient.isConnected()) {
return false;
}
try {
return ftpClient.removeDirectory(ftpDirectory);
} catch (IOException e) {
e.printStackTrace();
return false;
}
}
/**
* <p>Title: 关闭链接</p>
*
* <p>Description: </p>
*
* @author Yang
* @date 2019年5月21
*/
public void close() {
try {
if (ftpClient != null && ftpClient.isConnected()) {
ftpClient.disconnect();
}
logger.info("成功关闭连接，服务器ip:" + this.server + ", 端口:" + this.port);
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 获取ftp上文件的InputStream
*
* @param ftpDirName ftp文件夹名称
* @param ftpFileName ftp文件名称
* @return
*/
public byte[] getInputStream(String ftpDirName, String ftpFileName) {
try {
String dir = new String(ftpDirName.getBytes("utf-8"), "iso-8859-1");
if (!ftpClient.changeWorkingDirectory(dir)) {
System.out.println("切换目录失败：" + ftpDirName);
return null;
}
// 一定要加上字符集指定，因为获取文件时有中文，会出现乱码而获取不到。
String fileName = new String(ftpFileName.getBytes("utf-8"), "iso-8859-1");
ftpClient.setFileType(FTPClient.BINARY_FILE_TYPE);
// 每次数据连接之前，ftp client告诉ftp server开通一个端口来传输数据，ftp server可能每次开启不同的端口来传输数据，
// 但是在Linux上，由于安全限制，可能某些端口没有开启，所以就出现阻塞。
ftpClient.enterLocalPassiveMode();
InputStream inputStream = ftpClient.retrieveFileStream(fileName);
byte[] bytes = IOUtils.toByteArray(inputStream);
if (inputStream != null) {
inputStream.close();
}
ftpClient.getReply();
return bytes;
} catch (Exception e) {
log.error("获取文件流出现异常", e);
return null;
}
}
}
解压
/**
@Override
public void run(String... args) {
new Thread() {
@SneakyThrows
public void run() {
unzip();
}
}.start();
}
*/
public void unzip() throws Exception {
boolean open = ftpUtils.open();
while (open) {
FTPFile[] fileList = ftpUtils.getFileList("/");
if (null != fileList && fileList.length > 0) {
ZipInputStream zipInputStream = null;
for (FTPFile ftpFile : fileList) {
try {
byte[] fileByte = ftpUtils.getInputStream("/", ftpFile.getName());
InputStream fileStream = new ByteArrayInputStream(fileByte);
zipInputStream = new ZipInputStream(new BufferedInputStream(fileStream), Charset.forName("utf-8"));
while ((zipInputStream.getNextEntry()) != null) {
StringBuffer stringBuffer = new StringBuffer();
//读取
BufferedReader br = new BufferedReader(new InputStreamReader(zipInputStream));
String readlinestr;
//内容不为空，输出
while ((readlinestr = br.readLine()) != null) {
stringBuffer.append(readlinestr);
}
//结果处理
}
//方法关闭当前的ZIP条目并定位流以读取下一个条目
zipInputStream.closeEntry();
ftpUtils.deleteFile(ftpFile.getName());
} catch (Exception ex) {
log.error("数据解析错误{}", ex.getMessage());
break;
} finally {
if (zipInputStream != null) {
//关闭正在使用的文件删除文件
zipInputStream.close();
}
}
}
} else {
ftpUtils.close();
Thread.sleep(10000);
open = ftpUtils.open();
}
}
ftpUtils.close();
Thread.sleep(10000);
unzip();
}
```
```java
public void compress(String name) throws IOException {
for (int i =0;i<10000;i++){
System.currentTimeMillis();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
String fileName = name + ".zip";
String data = "{\n" +
" \"id\":\"1366946352243757058\",\n" +
" \"status\":3,\n" +
" \"updateTime\":1614740660000,\n" +
" \"version\":4\n" +
" }";
InputStream compressed = UnZipFileUtil.getCompressed(name + ".txt", data);
boolean open = ftpUtils.open();
ftpUtils.storeFile(new String((name+i + ".zip").getBytes(), "utf-8"), compressed);
}
}```
上一篇： Python入门篇之函数
下一篇： zip解压
推荐阅读