kirillius
/
mikrotik-container-backup-utility
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Промежуточный коммит

This commit is contained in:
kirillius 2025-02-06 15:33:07 +03:00
parent c69f90c7de
commit ea937b12f3
3 changed files with 217 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

393
src/main/java/ru/kirillius/mktbk/App.java
View File

@ -7,6 +7,7 @@ import org.apache.commons.compress.archivers.tar.TarConstants;
import ru.kirillius.utils.logging.SystemLogger; import ru.kirillius.utils.logging.SystemLogger;
import java.io.*; import java.io.*;
import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.logging.Level; import java.util.logging.Level;
@ -14,174 +15,211 @@ import java.util.logging.Level;
public class App { public class App {
private static final String LOG_CONTEXT = App.class.getSimpleName(); private static final String LOG_CONTEXT = App.class.getSimpleName();
private List<ContainerMetadata> containers; private List<ContainerMetadata> containers;
private Console console; private final Console console;
private DeviceContext deviceContext;
public static void main(String[] args) throws IOException, MikrotikApiException, InterruptedException { public App() {
console = new Console();
try {
auth();
while (!Thread.interrupted()) {
try {
if (!selectMode()) {
break;
}
} catch (Exception e) {
SystemLogger.error("Unexpected error", LOG_CONTEXT, e);
console.pause();
}
}
} finally {
try {
console.close();
} catch (IOException e) {
SystemLogger.error("Error closing console", LOG_CONTEXT, e);
}
}
}
public static void main(String[] args) {
SystemLogger.initializeLogging(Level.INFO, Collections.emptyList()); SystemLogger.initializeLogging(Level.INFO, Collections.emptyList());
new App(); new App();
} }
private ContainerMetadata selectContainer() {
// System.out.println("Found containers:"); private boolean selectMode() throws MikrotikApiException {
// var i = 0; var mode = console.select("What do you want to do?", List.of("Backup containers", "Restore backup", "Exit application"));
// for (var container : containers) { if (mode == 0) {
// System.out.println(String.valueOf(++i) + ' ' + container); selectWhatToBackup();
// } } else if (mode == 1) {
// do { selectWhatToRestore();
// } else {
// var index = -1; return false;
// }
// if (containers.size() > 1) { return true;
// do {
// index = -1;
// System.out.print("Select container you want to backup:");
// try {
// index = Integer.parseInt(cin()) - 1;
// } catch (NumberFormatException e) {
// SystemLogger.error("Invalid number", LOG_CONTEXT, e);
// }
// } while (index >= containers.size() || index < 0);
// }
//
// if (containers.isEmpty()) {
// return null;
// }
//
// var container = containers.get(index);
// System.out.print("Do you really want to backup container " + container + "? [y/n]");
//
// if (cin().equals("y")) {
// return container;
// }
// } while (true);
return null;
} }
public App() throws IOException, MikrotikApiException, InterruptedException { private void selectWhatToRestore() {
// try { //TODO
// console = new Console();
// auth();
// var mode = console.select("What do you want to do?", List.of("Backup containers", "Restore backup", "Exit application"));
// if(mode == 0){
// selectWhatToBackup();
// }else if(mode == 1){
// selectWhatToRestore();
// }
//
// containers = deviceContext.getContainers();
//
//
// ContainerMetadata container;
// do {
// container = selectContainer();
// if (container == null) {
// return;
// }
//
// if (!container.isRunning()) {
// SystemLogger.warning("The selected container is not running. There is a limitation that requires the container to be running.", LOG_CONTEXT);
// SystemLogger.message("Starting container " + container, LOG_CONTEXT);
// deviceContext.getApiConnection().execute("/container/start .id=" + container.getId());
// try {
// Thread.sleep(10000L);
// } catch (InterruptedException e) {
// throw new RuntimeException(e);
// }
//
// containers = deviceContext.getContainers();
// var id = container.getId();
// container = containers.stream().filter(containerMetadata -> containerMetadata.getId().equals(id)).findFirst().orElse(null);
// if (container == null || !container.isRunning()) {
// SystemLogger.error("Something went wrong. Unable to start container.", LOG_CONTEXT);
// pause();
// }
// }
// } while (container == null);
//
// SystemLogger.warning("/!\\ We are going to make a backup. You have to stop all services in this container to prevent data corruption.", LOG_CONTEXT);
// System.out.print("Type yes to continue or Ctrl+C to abort: ");
// while (!"yes".equals(cin())) ;
//
//
// var layerFile = new File(container.getGuid());
// var outputFile = new File("backup_of_" + container.getComment() + ".tar");
//
//
// var files = findFiles(container);
// downloadFiles(container, files, layerFile);
//
//
// SystemLogger.message("Building docker image " + outputFile.getName(), LOG_CONTEXT);
// var builder = new DockerImageBuilder(container);
// builder.build(layerFile, outputFile);
// SystemLogger.message("Backup is done ", LOG_CONTEXT);
// pause();
//
// } finally {
// if (reader != null) {
// reader.close();
// }
// if (deviceContext != null) {
// deviceContext.close();
// }
// }
// openSFTP();
} }
private void downloadFiles(ContainerMetadata container, List<FileMetadata> files, File outputFile) throws IOException { private void selectWhatToBackup() throws MikrotikApiException {
updateContainers();
var items = new ArrayList<>(containers.stream().map(ContainerMetadata::toString).toList());
items.add("Exit");
var selected = console.select("Select container you want to backup", items);
if (selected >= containers.size()) {
return;
}
backupContainer(containers.get(selected));
}
private void backupContainer(ContainerMetadata container) {
startContainerIfNotRunning(container);
container = validate(container);
if (!console.confirm("/!\\ We are going to make a backup. You have to stop all services in selected container to prevent data corruption.")) {
return;
}
var layerFile = new File(container.getGuid());
var suggestion = "backup_of_" + container.getComment();
var backupName = console.prompt("Enter name of backup [" + suggestion + "]", true);
if (backupName.isEmpty()) {
backupName = suggestion;
}
var outputFile = new File(backupName + ".tar.gz");
List<FileMetadata> files;
try {
files = loadContainerFiletree(container);
} catch (IOException | InterruptedException e) {
throw new RuntimeException("Error loading container file tree", e);
}
stopContainer(container);
container = validate(container);
SystemLogger.message("Building image layer from remote files", LOG_CONTEXT); SystemLogger.message("Building image layer from remote files", LOG_CONTEXT);
var sftp = deviceContext.getSftpClient(); createSystemLayer(container, files, layerFile);
try (var outputStream = new FileOutputStream(outputFile)) { SystemLogger.message("Download of " + files.size() + " has been completed", LOG_CONTEXT);
try (var tar = new TarArchiveOutputStream(outputStream)) {
tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); SystemLogger.message("Building docker image " + outputFile.getName(), LOG_CONTEXT);
var status = new DownloadStatus("Downloading", files.size()); var builder = new DockerImageBuilder(container);
for (var file : files) { try {
try { builder.build(layerFile, outputFile);
if (file.isDirectory()) { } catch (IOException e) {
var entry = new TarArchiveEntry(file.getPath(), TarConstants.LF_DIR); throw new RuntimeException(e);
entry.setGroupId(file.getGroupId()); } finally {
entry.setUserId(file.getUserId()); layerFile.delete();
entry.setMode(file.getMode()); }
tar.putArchiveEntry(entry); SystemLogger.message("Backup is done ", LOG_CONTEXT);
tar.closeArchiveEntry(); console.pause();
status.addCopiedFile(0);
} else if (file.isSymlink()) { }
var entry = new TarArchiveEntry(file.getPath(), TarConstants.LF_SYMLINK);
entry.setLinkName(file.getTarget()); private void stopContainer(ContainerMetadata container) {
tar.putArchiveEntry(entry); SystemLogger.message("Stopping container " + container, LOG_CONTEXT);
tar.closeArchiveEntry(); try {
status.addCopiedFile(0); deviceContext.getApiConnection().execute("/container/stop .id=" + container.getId());
} else { Thread.sleep(10000L);
var entry = new TarArchiveEntry(file.getPath()); } catch (MikrotikApiException | InterruptedException e) {
entry.setGroupId(file.getGroupId()); throw new RuntimeException("Error stopping container", e);
entry.setUserId(file.getUserId()); }
entry.setMode(file.getMode());
var remotePath = "/" + container.getGuid() + file.getPath(); }
var size = sftp.size(remotePath);
entry.setSize(size); private void startContainerIfNotRunning(ContainerMetadata container) {
tar.putArchiveEntry(entry); try {
sftp.get(remotePath, new SFTPFileStream(size, tar)); if (!container.isRunning()) {
tar.closeArchiveEntry(); SystemLogger.warning("The selected container is not running. There is a limitation that requires the container to be running.", LOG_CONTEXT);
status.addCopiedFile(size); console.pause();
} SystemLogger.message("Starting container " + container, LOG_CONTEXT);
} catch (IOException e) { deviceContext.getApiConnection().execute("/container/start .id=" + container.getId());
status.clear(); Thread.sleep(10000L);
SystemLogger.error("Unable to copy file " + file.getPath(), LOG_CONTEXT, e); updateContainers();
console.pause(); var id = container.getId();
} container = containers.stream().filter(containerMetadata -> containerMetadata.getId().equals(id)).findFirst().orElse(null);
if (container == null || !container.isRunning()) {
throw new RuntimeException("Something went wrong. Unable to start container.");
} }
tar.finish();
status.finish();
SystemLogger.message("Download of " + files.size() + " has been completed", LOG_CONTEXT);
} }
} catch (Exception e) {
throw new RuntimeException(e);
} }
} }
private List<FileMetadata> findFiles(ContainerMetadata container) throws IOException, InterruptedException { private ContainerMetadata validate(ContainerMetadata container) {
var found = containers.stream().filter(containerMetadata -> containerMetadata.getGuid().equals(container.getGuid())).findFirst();
if (found.isEmpty()) {
throw new RuntimeException("Failed to find container with guid: " + container.getGuid());
}
return found.get();
}
private void updateContainers() throws MikrotikApiException {
containers = deviceContext.getContainers();
}
private void createSystemLayer(ContainerMetadata container, List<FileMetadata> files, File outputFile) {
try {
var sftp = deviceContext.getSftpClient();
try (var outputStream = new FileOutputStream(outputFile)) {
try (var tar = new TarArchiveOutputStream(outputStream)) {
tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
var status = new DownloadStatus("Downloading", files.size());
for (var file : files) {
try {
if (file.isDirectory()) {
var entry = new TarArchiveEntry(file.getPath(), TarConstants.LF_DIR);
entry.setGroupId(file.getGroupId());
entry.setUserId(file.getUserId());
entry.setMode(file.getMode());
tar.putArchiveEntry(entry);
tar.closeArchiveEntry();
status.addCopiedFile(0);
} else if (file.isSymlink()) {
var entry = new TarArchiveEntry(file.getPath(), TarConstants.LF_SYMLINK);
entry.setLinkName(file.getTarget());
tar.putArchiveEntry(entry);
tar.closeArchiveEntry();
status.addCopiedFile(0);
} else {
var entry = new TarArchiveEntry(file.getPath());
entry.setGroupId(file.getGroupId());
entry.setUserId(file.getUserId());
entry.setMode(file.getMode());
var remotePath = "/" + container.getGuid() + file.getPath();
var size = sftp.size(remotePath);
entry.setSize(size);
tar.putArchiveEntry(entry);
sftp.get(remotePath, new SFTPFileStream(size, tar));
tar.closeArchiveEntry();
status.addCopiedFile(size);
}
} catch (IOException e) {
status.clear();
SystemLogger.error("Unable to copy file " + file.getPath(), LOG_CONTEXT, e);
console.pause();
}
}
tar.finish();
status.finish();
}
}
} catch (IOException e) {
throw new RuntimeException("Error creating layer image", e);
}
}
private List<FileMetadata> loadContainerFiletree(ContainerMetadata container) throws
IOException, InterruptedException {
SystemLogger.message("Reading container filesystem metadata...", LOG_CONTEXT); SystemLogger.message("Reading container filesystem metadata...", LOG_CONTEXT);
var systemDirs = List.of("/sys/", "/proc/", "/dev/", "/mnt/", "/run/", "/tmp/"); var systemDirs = List.of("/sys/", "/proc/", "/dev/", "/mnt/", "/run/", "/tmp/");
var filterFiles = List.of("/.type"); var filterFiles = List.of("/.type");
@ -190,43 +228,28 @@ public class App {
return files; return files;
} }
private DeviceContext deviceContext;
private void auth() { private void auth() {
do { do {
// try { try {
// var host = console.prompt("Enter remote host", false);
// System.out.print("Enter remote host:"); var username = console.prompt("Enter username", false);
// var host = cin(); var password = console.prompt("Enter password", false);
// if (host.trim().isEmpty()) {
// throw new RuntimeException("Remote host is empty"); console.clear();
// }
// deviceContext = new DeviceContext(host, username, password);
// System.out.print("Enter username:"); deviceContext.getApiConnection();
// var username = cin(); } catch (Exception e) {
// if (username.trim().isEmpty()) { SystemLogger.error("Unable to connect", LOG_CONTEXT, e);
// throw new RuntimeException("Username is empty"); if (deviceContext != null) {
// } deviceContext.close();
// System.out.print("Enter password:"); }
// var password = cin(); deviceContext = null;
// if (password.trim().isEmpty()) { }
// throw new RuntimeException("Password is empty");
// }
//
// deviceContext = new DeviceContext(host, username, password);
// deviceContext.getApiConnection();
// } catch (Exception e) {
// SystemLogger.error("Unable to connect", LOG_CONTEXT, e);
// if (deviceContext != null) {
// deviceContext.close();
// }
// deviceContext = null;
// }
} while (deviceContext == null); } while (deviceContext == null);
SystemLogger.message("Connected", LOG_CONTEXT); SystemLogger.message("Connected", LOG_CONTEXT);
} }
} }

10
src/main/java/ru/kirillius/mktbk/DeviceContext.java
View File

@ -206,11 +206,15 @@ public class DeviceContext implements Closeable {
return sshClient; return sshClient;
} }
public ApiConnection getApiConnection() throws MikrotikApiException { public ApiConnection getApiConnection() {
if (apiConnection == null) { if (apiConnection == null) {
SystemLogger.message("Initializing api connection", LOG_CONTEXT); SystemLogger.message("Initializing api connection", LOG_CONTEXT);
apiConnection = ApiConnection.connect(host); try {
apiConnection.login(username, password); apiConnection = ApiConnection.connect(host);
apiConnection.login(username, password);
} catch (MikrotikApiException e) {
throw new RuntimeException(e);
}
} }
return apiConnection; return apiConnection;
} }

3
src/main/java/ru/kirillius/mktbk/DockerImageBuilder.java
View File

@ -14,6 +14,7 @@ import java.nio.file.Files;
import java.security.MessageDigest; import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.zip.GZIPOutputStream;
public class DockerImageBuilder { public class DockerImageBuilder {
private final MessageDigest digest; private final MessageDigest digest;
@ -42,7 +43,7 @@ public class DockerImageBuilder {
} }
public void build(File layerFile, File outputFile) throws IOException { public void build(File layerFile, File outputFile) throws IOException {
try (var gzip = (new FileOutputStream(outputFile))) { try (var gzip = new GZIPOutputStream(new FileOutputStream(outputFile))) {
try (var tar = new TarArchiveOutputStream(gzip)) { try (var tar = new TarArchiveOutputStream(gzip)) {
var layer = addLayer(layerFile, tar); var layer = addLayer(layerFile, tar);
var repos = new JSONObject(); var repos = new JSONObject();