From 5ebe27715addc2c29c6d8969cf218677fe83b04c Mon Sep 17 00:00:00 2001 From: RJ-star <63181527+RJ-star@users.noreply.github.com> Date: Fri, 24 May 2024 16:09:36 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=95=B0=E6=8D=AE=E9=9B=86?= =?UTF-8?q?=E8=A7=A3=E5=8E=8B=E3=80=81=E6=95=B0=E6=8D=AE=E9=9B=86=E5=A2=9E?= =?UTF-8?q?=E5=B9=BF=E3=80=81=E4=BB=BB=E5=8A=A1=E6=8C=89=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E5=80=92=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/DatasetController.java | 55 ++++++++++- .../request/augmentDatasetRequest.java | 13 +++ .../springboot/service/DatasetService.java | 3 + .../service/impl/DatasetServiceImpl.java | 98 +++++++++++++++++++ .../service/impl/FileInfoServiceImpl.java | 5 +- .../service/impl/ODTaskServiceImpl.java | 2 + .../service/impl/RLTaskServiceImpl.java | 1 + .../service/impl/SATaskServiceImpl.java | 1 + src/main/resources/augment.py | 66 +++++++++++++ 9 files changed, 240 insertions(+), 4 deletions(-) create mode 100644 src/main/java/com/buaa/springboot/request/augmentDatasetRequest.java create mode 100644 src/main/resources/augment.py diff --git a/src/main/java/com/buaa/springboot/controller/DatasetController.java b/src/main/java/com/buaa/springboot/controller/DatasetController.java index 6eedbc7..c2112bb 100644 --- a/src/main/java/com/buaa/springboot/controller/DatasetController.java +++ b/src/main/java/com/buaa/springboot/controller/DatasetController.java @@ -2,6 +2,7 @@ package com.buaa.springboot.controller; import com.buaa.springboot.entity.Dataset; import com.buaa.springboot.entity.base.FileInfo; +import com.buaa.springboot.request.augmentDatasetRequest; import com.buaa.springboot.request.createDatasetRequest; import com.buaa.springboot.request.updateDatasetRequest; import com.buaa.springboot.service.FileInfoService; @@ -11,6 +12,10 @@ import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; import java.time.Instant; import java.util.Date; import java.util.List; @@ -39,14 +44,54 @@ public class DatasetController { return new ResponseEntity<>(dataset, HttpStatus.OK); } - //TODO 有待实现json传参实现 @PostMapping("/create") public ResponseEntity createODDataset(@RequestBody createDatasetRequest request) { + + // TODO 直接使用winrar.exe的路径 Dataset dataset = new Dataset(); FileInfo fileInfo = fileInfoService.getFileInfoById(request.getFileId()); + File file = new File(fileInfo.getRelativePhysicsPath()); + + //解压 + //先创建目标文件夹 + String folderPath = file.getParent(); + String name = file.getName(); + name = name.substring(0, name.lastIndexOf('.')); + folderPath = folderPath + "/" + name; + File dir = new File(folderPath); + if (!dir.exists()) { + if (dir.mkdirs()) { + System.out.println("创建目录成功"); + } + } + + String winrarPath = "C:\\Program Files\\WinRAR\\WinRAR.exe"; + String zipFilePath = fileInfo.getRelativePhysicsPath(); + String destinationPath = dir.toString(); + try { + // 创建一个ProcessBuilder实例并设置命令 + ProcessBuilder builder = new ProcessBuilder(winrarPath, "x", zipFilePath, destinationPath); + + // 启动进程 + Process process = builder.start(); + + // 从进程的输入流中读取输出 + BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line; + while ((line = reader.readLine()) != null) { + System.out.println(line); + } + + // 等待进程结束并获取退出码 + int exitCode = process.waitFor(); + System.out.println("Exited with code: " + exitCode); + } catch (Exception e) { + e.printStackTrace(); + } + int lastIndex = fileInfo.getRelativePhysicsPath().lastIndexOf("."); dataset.setName(request.getName()); dataset.setText(request.getText()); - dataset.setPath(fileInfo.getRelativePhysicsPath()); + dataset.setPath(fileInfo.getRelativePhysicsPath().substring(0, lastIndex)); dataset.setSize(fileInfo.getSize()); Instant date = Instant.now(); dataset.setCreateTime(Date.from(date)); @@ -68,4 +113,10 @@ public class DatasetController { datasetService.deleteDataset(id); return ResponseEntity.ok("数据集删除成功"); } + + @PostMapping("/augmentation") + public ResponseEntity augmentDataset(@RequestBody augmentDatasetRequest request) { + datasetService.augmentDataset(request.getId(),request.getMethod(),request.getParameters()); + return ResponseEntity.ok("增广成功"); + } } diff --git a/src/main/java/com/buaa/springboot/request/augmentDatasetRequest.java b/src/main/java/com/buaa/springboot/request/augmentDatasetRequest.java new file mode 100644 index 0000000..9f1ce04 --- /dev/null +++ b/src/main/java/com/buaa/springboot/request/augmentDatasetRequest.java @@ -0,0 +1,13 @@ +package com.buaa.springboot.request; + +import io.swagger.models.auth.In; +import lombok.Getter; + +import java.util.Map; + +@Getter +public class augmentDatasetRequest { + private Integer id; + private String method; + private Map parameters; +} diff --git a/src/main/java/com/buaa/springboot/service/DatasetService.java b/src/main/java/com/buaa/springboot/service/DatasetService.java index 52996b3..19cf8fd 100644 --- a/src/main/java/com/buaa/springboot/service/DatasetService.java +++ b/src/main/java/com/buaa/springboot/service/DatasetService.java @@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.extension.service.IService; import com.buaa.springboot.entity.Dataset; import java.util.List; +import java.util.Map; public interface DatasetService extends IService { public List getAllDatasets(); @@ -15,4 +16,6 @@ public interface DatasetService extends IService { public void updateDataset(int id,String name,String text); public void deleteDataset(int id); + + public String augmentDataset(int id, String method, Map parameters); } diff --git a/src/main/java/com/buaa/springboot/service/impl/DatasetServiceImpl.java b/src/main/java/com/buaa/springboot/service/impl/DatasetServiceImpl.java index e21a6d7..2773255 100644 --- a/src/main/java/com/buaa/springboot/service/impl/DatasetServiceImpl.java +++ b/src/main/java/com/buaa/springboot/service/impl/DatasetServiceImpl.java @@ -8,8 +8,21 @@ import com.buaa.springboot.service.DatasetService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.time.Instant; +import java.time.LocalTime; +import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; @Service public class DatasetServiceImpl extends ServiceImpl implements DatasetService { @@ -56,4 +69,89 @@ public class DatasetServiceImpl extends ServiceImpl impl existingDataset.setIsDeleted(1); datasetMapper.updateById(existingDataset); } + + @Override + public String augmentDataset(int id, String method, Map parameters) { + Dataset oldDataset = datasetMapper.selectById(id); + String path = oldDataset.getPath(); + int size = parameters.size(); + List commandList = new ArrayList<>(); +// TODO 拼接路径！！rootPath和conda的python路径 + String rootPath = "F:\\evaluate_backend\\src\\main\\resources"; + commandList.add("C:\\Users\\G304\\anaconda3\\envs\\pythonProject1\\python.exe"); + commandList.add(rootPath + "\\augment.py"); + commandList.add(path); + commandList.add(method); + for (int i = 1; i <= size; i++) { + String pos = "parameter" + String.valueOf(i); + commandList.add(parameters.get(pos)); + } + String newPath = ""; + try { + ProcessBuilder builder = new ProcessBuilder(commandList); + builder.redirectErrorStream(true); + Process process = builder.start(); + BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + newPath = reader.readLine(); + } catch (Exception e) { + e.printStackTrace(); + } + System.out.println(newPath); + //获取新数据集名 + int firstIndex = newPath.lastIndexOf("\\"); + String s1 = newPath.substring(firstIndex); + int secondIndex = s1.indexOf("-"); + s1 = s1.substring(secondIndex); + + Instant date = Instant.now(); + Dataset dataset = new Dataset(); + dataset.setName(oldDataset.getName() + "-" + s1) + .setText("") + .setCreateTime(Date.from(date)) + .setModifyTime(Date.from(date)) + .setSize(oldDataset.getSize()) + .setSource(oldDataset.getId()) + .setPath(newPath) + .setIsDeleted(0); + save(dataset); + //TODO label复制过去 + copy(oldDataset.getPath() + "\\labels", newPath + "\\labels"); + //TODO 增广算法有问题 + return ""; + } + + public void copy(String oldPath, String newPath) { + Path sourceDir = Paths.get(oldPath); + Path targetDir = Paths.get(newPath); + + try { + // 检查目标目录是否存在，不存在则创建 + if (!Files.exists(targetDir)) { + Files.createDirectories(targetDir); + } + + // 使用 Files.walk 方法遍历源目录 + Files.walk(sourceDir) + .forEach(sourcePath -> { + try { + // 计算目标路径 + Path targetPath = targetDir.resolve(sourceDir.relativize(sourcePath)); + + // 如果是目录，则创建目录 + if (Files.isDirectory(sourcePath)) { + if (!Files.exists(targetPath)) { + Files.createDirectory(targetPath); + } + } else { + // 如果是文件，则复制文件 + Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING); + } + } catch (IOException e) { + e.printStackTrace(); + } + }); + } catch (IOException e) { + e.printStackTrace(); + } + } } diff --git a/src/main/java/com/buaa/springboot/service/impl/FileInfoServiceImpl.java b/src/main/java/com/buaa/springboot/service/impl/FileInfoServiceImpl.java index 0243e6f..8512bda 100644 --- a/src/main/java/com/buaa/springboot/service/impl/FileInfoServiceImpl.java +++ b/src/main/java/com/buaa/springboot/service/impl/FileInfoServiceImpl.java @@ -28,7 +28,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; public class FileInfoServiceImpl extends ServiceImpl implements FileInfoService { @Autowired private FileInfoMapper fileInfoMapper; - @Value("${file.root-path}") + @Value("F:\\test") private String rootPath; private static final String SEPARATOR = "/"; @@ -74,7 +74,8 @@ public class FileInfoServiceImpl extends ServiceImpl i initFileName = initFileName.substring(initFileName.lastIndexOf(SEPARATOR) + 1); // IE浏览器有个神奇的bug，就是他的文件名上传时包含了此文件在本地的路径。 // 所以此处做了截取文件名的操作 - String filePath = rootPath + "/files/"; + // TODO 要先手动新建files路径 + String filePath = rootPath + "\\files\\"; try { byte[] bytes = file.getBytes(); //前边上传文件的版本 diff --git a/src/main/java/com/buaa/springboot/service/impl/ODTaskServiceImpl.java b/src/main/java/com/buaa/springboot/service/impl/ODTaskServiceImpl.java index aa41164..deff252 100644 --- a/src/main/java/com/buaa/springboot/service/impl/ODTaskServiceImpl.java +++ b/src/main/java/com/buaa/springboot/service/impl/ODTaskServiceImpl.java @@ -1,5 +1,6 @@ package com.buaa.springboot.service.impl; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import com.buaa.springboot.entity.ODTask; @@ -31,6 +32,7 @@ public class ODTaskServiceImpl extends ServiceImpl impleme public List getAllODTasks(){ QueryWrapper queryWrapper = new QueryWrapper<>(); queryWrapper.eq("is_deleted", 0); // 添加条件，等价于 isDeleted = 0 + queryWrapper.orderByDesc("modify_time"); return odTaskMapper.selectList(queryWrapper); } diff --git a/src/main/java/com/buaa/springboot/service/impl/RLTaskServiceImpl.java b/src/main/java/com/buaa/springboot/service/impl/RLTaskServiceImpl.java index 91eaef9..df3f2b9 100644 --- a/src/main/java/com/buaa/springboot/service/impl/RLTaskServiceImpl.java +++ b/src/main/java/com/buaa/springboot/service/impl/RLTaskServiceImpl.java @@ -31,6 +31,7 @@ public class RLTaskServiceImpl extends ServiceImpl impleme public List getAllRLTasks(){ QueryWrapper queryWrapper = new QueryWrapper<>(); queryWrapper.eq("is_deleted", 0); // 添加条件，等价于 isDeleted = 0 + queryWrapper.orderByDesc("modify_time"); return rlTaskMapper.selectList(queryWrapper); } diff --git a/src/main/java/com/buaa/springboot/service/impl/SATaskServiceImpl.java b/src/main/java/com/buaa/springboot/service/impl/SATaskServiceImpl.java index a122b52..f4c9853 100644 --- a/src/main/java/com/buaa/springboot/service/impl/SATaskServiceImpl.java +++ b/src/main/java/com/buaa/springboot/service/impl/SATaskServiceImpl.java @@ -31,6 +31,7 @@ public class SATaskServiceImpl extends ServiceImpl impleme public List getAllSATasks(){ QueryWrapper queryWrapper = new QueryWrapper<>(); queryWrapper.eq("is_deleted", 0); // 添加条件，等价于 isDeleted = 0 + queryWrapper.orderByDesc("modify_time"); return rlTaskMapper.selectList(queryWrapper); } diff --git a/src/main/resources/augment.py b/src/main/resources/augment.py new file mode 100644 index 0000000..bbe7ff2 --- /dev/null +++ b/src/main/resources/augment.py @@ -0,0 +1,66 @@ +import argparse +import os +import datetime +from PIL import Image, ImageOps +import numpy as np +import random + + +def add_gaussian_noise(image, mean=0.0, std=1.0): + np_image = np.array(image) + noise = np.random.normal(mean, std, np_image.shape) + noisy_image = np.clip(np_image + noise, 0, 255).astype(np.uint8) + return Image.fromarray(noisy_image) + + +def add_salt_and_pepper_noise(image, amount=0.01): + np_image = np.array(image) + row, col, _ = np_image.shape + num_salt = np.ceil(amount * row * col) + num_pepper = np.ceil(amount * row * col) + + # Add Salt noise + coords = [np.random.randint(0, i - 1, int(num_salt)) for i in np_image.shape] + np_image[coords[0], coords[1], :] = 1 + + # Add Pepper noise + coords = [np.random.randint(0, i - 1, int(num_pepper)) for i in np_image.shape] + np_image[coords[0], coords[1], :] = 0 + + return Image.fromarray(np_image) + + +def process_images(input_path, method, param): + output_path = f"{input_path}-{method}-{param}-{datetime.datetime.now().strftime('%Y%m%d%H%M%S')}" + print(output_path) + output_path = output_path + "\\images" + os.makedirs(output_path, exist_ok=True) + + for filename in os.listdir(input_path + "\\images"): + if filename.endswith(('.png', '.jpg', '.jpeg', '.bmp', '.tif', '.tiff')): + image_path = os.path.join(input_path, "images", filename) + image = Image.open(image_path) + + if method == 1: + processed_image = add_gaussian_noise(image, std=param) + output_file = os.path.join(output_path, f"{filename}") + elif method == 2: + processed_image = add_salt_and_pepper_noise(image, amount=param) + output_file = os.path.join(output_path, f"{filename}") + else: + # print(f"Unknown method: {method}") + continue + + processed_image.save(output_file) + # print(f"Saved processed image to {output_file}") + + +parser = argparse.ArgumentParser(description="Process a dataset of images with specified noise.") +parser.add_argument('path', type=str, help='Path to the image dataset') +# parser.add_argument('output_path', type=str, help='output path to the image dataset') +parser.add_argument('method', type=int, help='Method type: 1 for Gaussian noise, 2 for Salt and Pepper noise') +parser.add_argument('param', type=float, help='Parameter for the noise method') + +args = parser.parse_args() + +process_images(args.path, args.method, args.param) -- Gitee