package com.bcxin.file; import cn.hutool.core.io.FileUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.http.HttpRequest; import cn.hutool.http.HttpResponse; import cn.hutool.http.HttpUtil; import java.io.*; import java.net.HttpURLConnection; import java.net.URL; import java.util.ArrayList; import java.util.List; import java.util.function.Consumer; import java.util.function.Supplier; public class FileUtils { /** * 根据url地址下载文件 * @param fileUrl 文件地址 * @param localFilePath 保存本地路径 */ public static boolean downloadFile(String fileUrl, String localFilePath) { URL urlfile = null; try { urlfile = new URL(fileUrl); String filePath = null; String path = urlfile.getPath(); System.out.println("path--->" + fileUrl); if (path.equals("/getResource.do")) { //判断地址是http://bj.baibaodun.cn:8801/ars-web/getResource.do?path=upload/2020-08-28/1598607857060.zip 就截取 =后面的路径 filePath = fileUrl.substring(fileUrl.indexOf("=") + 1); } else if (fileUrl.contains("=")) { //截取双路径如 http://v5qy.baibaodun.cn/obpm/v2/sync/file/download?f=https%3A%2F%2Fbcxin-v5-prod.obs.cn-north-1.myhuaweicloud.com%2Fuploads%2F2022%2F07-12%2F658979513459277824--__AOkTvsah9VLicJhQ1FO%2F1657606996828%2F%E4%BF%9D%E5%AE%89%E6%9C%8D%E5%8A%A1%E5%85%AC%E5%8F%B8%E8%AE%BE%E7%AB%8B%E5%88%86%E5%85%AC%E5%8F%B8%E5%A4%87%E6%A1%88.zip if (fileUrl.substring(fileUrl.indexOf("=") + 1).contains("https")) { filePath = fileUrl.substring(fileUrl.indexOf("=") + 1).substring(fileUrl.substring(fileUrl.indexOf("=") + 1).indexOf("com") + 3); } else { if (fileUrl.substring(fileUrl.indexOf("=") + 1).indexOf("/") != 0) { filePath = "/" + fileUrl.substring(fileUrl.indexOf("=") + 1); } else { filePath = fileUrl.substring(fileUrl.indexOf("=") + 1); } } } else { filePath = path; } filePath = filePath.replaceAll("/upload/", "/uploads/"); filePath = "/uploads/" + filePath.replace("/uploads/", ""); filePath = filePath.replaceAll("/+", "/"); System.out.println("filePath:===" + filePath); File f = new File(localFilePath + filePath); if (!f.getParentFile().exists() && !f.getParentFile().isDirectory()) { FileUtil.mkdir(f.getParentFile()); //f.getParentFile().mkdirs(); } HttpUtil.createGet(fileUrl, true) .header("accessToken", "eyJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJhdXRoMCIsImV4cCI6MTY2Njg2MDAwMSwidXNlcm5hbWUiOiI3TkVMR1RYaSJ9.Fvx16149p8Grp-a_QwFRGro_sWD1xqL2QV5beL9bRcc") .executeAsync().writeBody(new File(localFilePath+filePath),null); } catch (Exception e) { e.printStackTrace(); return false; } return true; } public static List readFile(String path, String errorPath, Consumer> extractUrlsExecutor) throws IOException { List pathList = new ArrayList<>(); try { InputStreamReader reader = new InputStreamReader( new FileInputStream( path ), "UTF-8" ); BufferedReader bfReader = new BufferedReader( reader ); String tmpContent = null; while ( ( tmpContent = bfReader.readLine() ) != null ) { tmpContent = filter(tmpContent); if(StrUtil.isNotEmpty(tmpContent)) { if(tmpContent.startsWith("http") && !tmpContent.startsWith("https://") && !tmpContent.startsWith("http://")) { tmpContent = tmpContent.replace("http:/","http://").replace("https:/","https://"); } if(!tmpContent.startsWith("https://") && !tmpContent.startsWith("http://")){ try { FileUtil.appendString(tmpContent, errorPath, "UTF-8"); } catch (Exception ex) { ex.printStackTrace(); } continue; } tmpContent = tmpContent.replaceAll("%2F", "/"); if(tmpContent.contains("myhuaweicloud.com/obpm/uploads/")){ tmpContent = tmpContent.replace("obpm/uploads/",""); } pathList.add(tmpContent); if(pathList.size()%1000==0) { extractUrlsExecutor.accept(pathList); pathList.clear(); } } } if(pathList.size()>0) { extractUrlsExecutor.accept(pathList); pathList.clear(); } bfReader.close(); } catch ( UnsupportedEncodingException e ) { // 忽略 e.printStackTrace(); } return pathList; } // 过滤输入字符串, 剔除多行注释以及替换掉反斜杠 private static String filter ( String input ) { return input.replaceAll( "/\\*[\\s\\S]*?\\*/", "" ); } public static List getAllFilePaths(File[] fileList) { List pathList = new ArrayList<>(); if(fileList != null) { for (File file : fileList) { if (file.getName().endsWith(".txt")) { pathList.add(file.getAbsolutePath()); } } } return pathList; } }