From ea937b12f3c89b8639ae790566d1b26d28d9bd24 Mon Sep 17 00:00:00 2001 From: kirillius Date: Thu, 6 Feb 2025 15:33:07 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9F=D1=80=D0=BE=D0=BC=D0=B5=D0=B6=D1=83?= =?UTF-8?q?=D1=82=D0=BE=D1=87=D0=BD=D1=8B=D0=B9=20=D0=BA=D0=BE=D0=BC=D0=BC?= =?UTF-8?q?=D0=B8=D1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/ru/kirillius/mktbk/App.java | 393 +++++++++--------- .../ru/kirillius/mktbk/DeviceContext.java | 10 +- .../kirillius/mktbk/DockerImageBuilder.java | 3 +- 3 files changed, 217 insertions(+), 189 deletions(-) diff --git a/src/main/java/ru/kirillius/mktbk/App.java b/src/main/java/ru/kirillius/mktbk/App.java index e3d89d5..b08a7fc 100644 --- a/src/main/java/ru/kirillius/mktbk/App.java +++ b/src/main/java/ru/kirillius/mktbk/App.java @@ -7,6 +7,7 @@ import org.apache.commons.compress.archivers.tar.TarConstants; import ru.kirillius.utils.logging.SystemLogger; import java.io.*; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.logging.Level; @@ -14,174 +15,211 @@ import java.util.logging.Level; public class App { private static final String LOG_CONTEXT = App.class.getSimpleName(); private List containers; - private Console console; + private final Console console; + private DeviceContext deviceContext; - public static void main(String[] args) throws IOException, MikrotikApiException, InterruptedException { + public App() { + + + console = new Console(); + try { + auth(); + while (!Thread.interrupted()) { + try { + if (!selectMode()) { + break; + } + } catch (Exception e) { + SystemLogger.error("Unexpected error", LOG_CONTEXT, e); + console.pause(); + } + } + + } finally { + try { + console.close(); + } catch (IOException e) { + SystemLogger.error("Error closing console", LOG_CONTEXT, e); + } + } + + + } + + public static void main(String[] args) { SystemLogger.initializeLogging(Level.INFO, Collections.emptyList()); - new App(); } - private ContainerMetadata selectContainer() { -// System.out.println("Found containers:"); -// var i = 0; -// for (var container : containers) { -// System.out.println(String.valueOf(++i) + ' ' + container); -// } -// do { -// -// var index = -1; -// -// if (containers.size() > 1) { -// do { -// index = -1; -// System.out.print("Select container you want to backup:"); -// try { -// index = Integer.parseInt(cin()) - 1; -// } catch (NumberFormatException e) { -// SystemLogger.error("Invalid number", LOG_CONTEXT, e); -// } -// } while (index >= containers.size() || index < 0); -// } -// -// if (containers.isEmpty()) { -// return null; -// } -// -// var container = containers.get(index); -// System.out.print("Do you really want to backup container " + container + "? [y/n]"); -// -// if (cin().equals("y")) { -// return container; -// } -// } while (true); - return null; + + private boolean selectMode() throws MikrotikApiException { + var mode = console.select("What do you want to do?", List.of("Backup containers", "Restore backup", "Exit application")); + if (mode == 0) { + selectWhatToBackup(); + } else if (mode == 1) { + selectWhatToRestore(); + } else { + return false; + } + return true; } - public App() throws IOException, MikrotikApiException, InterruptedException { -// try { -// console = new Console(); -// auth(); -// var mode = console.select("What do you want to do?", List.of("Backup containers", "Restore backup", "Exit application")); -// if(mode == 0){ -// selectWhatToBackup(); -// }else if(mode == 1){ -// selectWhatToRestore(); -// } -// -// containers = deviceContext.getContainers(); -// -// -// ContainerMetadata container; -// do { -// container = selectContainer(); -// if (container == null) { -// return; -// } -// -// if (!container.isRunning()) { -// SystemLogger.warning("The selected container is not running. There is a limitation that requires the container to be running.", LOG_CONTEXT); -// SystemLogger.message("Starting container " + container, LOG_CONTEXT); -// deviceContext.getApiConnection().execute("/container/start .id=" + container.getId()); -// try { -// Thread.sleep(10000L); -// } catch (InterruptedException e) { -// throw new RuntimeException(e); -// } -// -// containers = deviceContext.getContainers(); -// var id = container.getId(); -// container = containers.stream().filter(containerMetadata -> containerMetadata.getId().equals(id)).findFirst().orElse(null); -// if (container == null || !container.isRunning()) { -// SystemLogger.error("Something went wrong. Unable to start container.", LOG_CONTEXT); -// pause(); -// } -// } -// } while (container == null); -// -// SystemLogger.warning("/!\\ We are going to make a backup. You have to stop all services in this container to prevent data corruption.", LOG_CONTEXT); -// System.out.print("Type yes to continue or Ctrl+C to abort: "); -// while (!"yes".equals(cin())) ; -// -// -// var layerFile = new File(container.getGuid()); -// var outputFile = new File("backup_of_" + container.getComment() + ".tar"); -// -// -// var files = findFiles(container); -// downloadFiles(container, files, layerFile); -// -// -// SystemLogger.message("Building docker image " + outputFile.getName(), LOG_CONTEXT); -// var builder = new DockerImageBuilder(container); -// builder.build(layerFile, outputFile); -// SystemLogger.message("Backup is done ", LOG_CONTEXT); -// pause(); -// -// } finally { -// if (reader != null) { -// reader.close(); -// } -// if (deviceContext != null) { -// deviceContext.close(); -// } -// } - -// openSFTP(); - - + private void selectWhatToRestore() { + //TODO } - private void downloadFiles(ContainerMetadata container, List files, File outputFile) throws IOException { + private void selectWhatToBackup() throws MikrotikApiException { + updateContainers(); + var items = new ArrayList<>(containers.stream().map(ContainerMetadata::toString).toList()); + items.add("Exit"); + var selected = console.select("Select container you want to backup", items); + if (selected >= containers.size()) { + return; + } + backupContainer(containers.get(selected)); + } + + private void backupContainer(ContainerMetadata container) { + startContainerIfNotRunning(container); + container = validate(container); + if (!console.confirm("/!\\ We are going to make a backup. You have to stop all services in selected container to prevent data corruption.")) { + return; + } + + var layerFile = new File(container.getGuid()); + + var suggestion = "backup_of_" + container.getComment(); + var backupName = console.prompt("Enter name of backup [" + suggestion + "]", true); + if (backupName.isEmpty()) { + backupName = suggestion; + } + + var outputFile = new File(backupName + ".tar.gz"); + + + List files; + try { + files = loadContainerFiletree(container); + } catch (IOException | InterruptedException e) { + throw new RuntimeException("Error loading container file tree", e); + } + + stopContainer(container); + container = validate(container); SystemLogger.message("Building image layer from remote files", LOG_CONTEXT); - var sftp = deviceContext.getSftpClient(); - try (var outputStream = new FileOutputStream(outputFile)) { - try (var tar = new TarArchiveOutputStream(outputStream)) { - tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); - var status = new DownloadStatus("Downloading", files.size()); - for (var file : files) { - try { - if (file.isDirectory()) { - var entry = new TarArchiveEntry(file.getPath(), TarConstants.LF_DIR); - entry.setGroupId(file.getGroupId()); - entry.setUserId(file.getUserId()); - entry.setMode(file.getMode()); - tar.putArchiveEntry(entry); - tar.closeArchiveEntry(); - status.addCopiedFile(0); - } else if (file.isSymlink()) { - var entry = new TarArchiveEntry(file.getPath(), TarConstants.LF_SYMLINK); - entry.setLinkName(file.getTarget()); - tar.putArchiveEntry(entry); - tar.closeArchiveEntry(); - status.addCopiedFile(0); - } else { - var entry = new TarArchiveEntry(file.getPath()); - entry.setGroupId(file.getGroupId()); - entry.setUserId(file.getUserId()); - entry.setMode(file.getMode()); - var remotePath = "/" + container.getGuid() + file.getPath(); - var size = sftp.size(remotePath); - entry.setSize(size); - tar.putArchiveEntry(entry); - sftp.get(remotePath, new SFTPFileStream(size, tar)); - tar.closeArchiveEntry(); - status.addCopiedFile(size); - } - } catch (IOException e) { - status.clear(); - SystemLogger.error("Unable to copy file " + file.getPath(), LOG_CONTEXT, e); - console.pause(); - } + createSystemLayer(container, files, layerFile); + SystemLogger.message("Download of " + files.size() + " has been completed", LOG_CONTEXT); + + SystemLogger.message("Building docker image " + outputFile.getName(), LOG_CONTEXT); + var builder = new DockerImageBuilder(container); + try { + builder.build(layerFile, outputFile); + } catch (IOException e) { + throw new RuntimeException(e); + } finally { + layerFile.delete(); + } + SystemLogger.message("Backup is done ", LOG_CONTEXT); + console.pause(); + + } + + private void stopContainer(ContainerMetadata container) { + SystemLogger.message("Stopping container " + container, LOG_CONTEXT); + try { + deviceContext.getApiConnection().execute("/container/stop .id=" + container.getId()); + Thread.sleep(10000L); + } catch (MikrotikApiException | InterruptedException e) { + throw new RuntimeException("Error stopping container", e); + } + + } + + private void startContainerIfNotRunning(ContainerMetadata container) { + try { + if (!container.isRunning()) { + SystemLogger.warning("The selected container is not running. There is a limitation that requires the container to be running.", LOG_CONTEXT); + console.pause(); + SystemLogger.message("Starting container " + container, LOG_CONTEXT); + deviceContext.getApiConnection().execute("/container/start .id=" + container.getId()); + Thread.sleep(10000L); + updateContainers(); + var id = container.getId(); + container = containers.stream().filter(containerMetadata -> containerMetadata.getId().equals(id)).findFirst().orElse(null); + if (container == null || !container.isRunning()) { + throw new RuntimeException("Something went wrong. Unable to start container."); } - tar.finish(); - status.finish(); - SystemLogger.message("Download of " + files.size() + " has been completed", LOG_CONTEXT); } + } catch (Exception e) { + throw new RuntimeException(e); } } - private List findFiles(ContainerMetadata container) throws IOException, InterruptedException { + private ContainerMetadata validate(ContainerMetadata container) { + var found = containers.stream().filter(containerMetadata -> containerMetadata.getGuid().equals(container.getGuid())).findFirst(); + if (found.isEmpty()) { + throw new RuntimeException("Failed to find container with guid: " + container.getGuid()); + } + return found.get(); + } + + private void updateContainers() throws MikrotikApiException { + containers = deviceContext.getContainers(); + } + + private void createSystemLayer(ContainerMetadata container, List files, File outputFile) { + try { + var sftp = deviceContext.getSftpClient(); + try (var outputStream = new FileOutputStream(outputFile)) { + try (var tar = new TarArchiveOutputStream(outputStream)) { + tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); + var status = new DownloadStatus("Downloading", files.size()); + for (var file : files) { + try { + if (file.isDirectory()) { + var entry = new TarArchiveEntry(file.getPath(), TarConstants.LF_DIR); + entry.setGroupId(file.getGroupId()); + entry.setUserId(file.getUserId()); + entry.setMode(file.getMode()); + tar.putArchiveEntry(entry); + tar.closeArchiveEntry(); + status.addCopiedFile(0); + } else if (file.isSymlink()) { + var entry = new TarArchiveEntry(file.getPath(), TarConstants.LF_SYMLINK); + entry.setLinkName(file.getTarget()); + tar.putArchiveEntry(entry); + tar.closeArchiveEntry(); + status.addCopiedFile(0); + } else { + var entry = new TarArchiveEntry(file.getPath()); + entry.setGroupId(file.getGroupId()); + entry.setUserId(file.getUserId()); + entry.setMode(file.getMode()); + var remotePath = "/" + container.getGuid() + file.getPath(); + var size = sftp.size(remotePath); + entry.setSize(size); + tar.putArchiveEntry(entry); + sftp.get(remotePath, new SFTPFileStream(size, tar)); + tar.closeArchiveEntry(); + status.addCopiedFile(size); + } + } catch (IOException e) { + status.clear(); + SystemLogger.error("Unable to copy file " + file.getPath(), LOG_CONTEXT, e); + console.pause(); + } + } + tar.finish(); + status.finish(); + } + } + } catch (IOException e) { + throw new RuntimeException("Error creating layer image", e); + } + } + + private List loadContainerFiletree(ContainerMetadata container) throws + IOException, InterruptedException { SystemLogger.message("Reading container filesystem metadata...", LOG_CONTEXT); var systemDirs = List.of("/sys/", "/proc/", "/dev/", "/mnt/", "/run/", "/tmp/"); var filterFiles = List.of("/.type"); @@ -190,43 +228,28 @@ public class App { return files; } - - private DeviceContext deviceContext; - private void auth() { do { -// try { -// -// System.out.print("Enter remote host:"); -// var host = cin(); -// if (host.trim().isEmpty()) { -// throw new RuntimeException("Remote host is empty"); -// } -// -// System.out.print("Enter username:"); -// var username = cin(); -// if (username.trim().isEmpty()) { -// throw new RuntimeException("Username is empty"); -// } -// System.out.print("Enter password:"); -// var password = cin(); -// if (password.trim().isEmpty()) { -// throw new RuntimeException("Password is empty"); -// } -// -// deviceContext = new DeviceContext(host, username, password); -// deviceContext.getApiConnection(); -// } catch (Exception e) { -// SystemLogger.error("Unable to connect", LOG_CONTEXT, e); -// if (deviceContext != null) { -// deviceContext.close(); -// } -// deviceContext = null; -// } + try { + var host = console.prompt("Enter remote host", false); + var username = console.prompt("Enter username", false); + var password = console.prompt("Enter password", false); + + console.clear(); + + deviceContext = new DeviceContext(host, username, password); + deviceContext.getApiConnection(); + } catch (Exception e) { + SystemLogger.error("Unable to connect", LOG_CONTEXT, e); + if (deviceContext != null) { + deviceContext.close(); + } + deviceContext = null; + } } while (deviceContext == null); SystemLogger.message("Connected", LOG_CONTEXT); } -} +} \ No newline at end of file diff --git a/src/main/java/ru/kirillius/mktbk/DeviceContext.java b/src/main/java/ru/kirillius/mktbk/DeviceContext.java index d2db4d7..040c753 100644 --- a/src/main/java/ru/kirillius/mktbk/DeviceContext.java +++ b/src/main/java/ru/kirillius/mktbk/DeviceContext.java @@ -206,11 +206,15 @@ public class DeviceContext implements Closeable { return sshClient; } - public ApiConnection getApiConnection() throws MikrotikApiException { + public ApiConnection getApiConnection() { if (apiConnection == null) { SystemLogger.message("Initializing api connection", LOG_CONTEXT); - apiConnection = ApiConnection.connect(host); - apiConnection.login(username, password); + try { + apiConnection = ApiConnection.connect(host); + apiConnection.login(username, password); + } catch (MikrotikApiException e) { + throw new RuntimeException(e); + } } return apiConnection; } diff --git a/src/main/java/ru/kirillius/mktbk/DockerImageBuilder.java b/src/main/java/ru/kirillius/mktbk/DockerImageBuilder.java index 5b9d085..87f1f48 100644 --- a/src/main/java/ru/kirillius/mktbk/DockerImageBuilder.java +++ b/src/main/java/ru/kirillius/mktbk/DockerImageBuilder.java @@ -14,6 +14,7 @@ import java.nio.file.Files; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.regex.Pattern; +import java.util.zip.GZIPOutputStream; public class DockerImageBuilder { private final MessageDigest digest; @@ -42,7 +43,7 @@ public class DockerImageBuilder { } public void build(File layerFile, File outputFile) throws IOException { - try (var gzip = (new FileOutputStream(outputFile))) { + try (var gzip = new GZIPOutputStream(new FileOutputStream(outputFile))) { try (var tar = new TarArchiveOutputStream(gzip)) { var layer = addLayer(layerFile, tar); var repos = new JSONObject();