From 0fae695941b3faf7841d19d79cec9170b7a1ecdb Mon Sep 17 00:00:00 2001 From: kirillius Date: Thu, 30 Jan 2025 22:26:14 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D0=BB=20?= =?UTF-8?q?=D1=84=D0=B8=D1=87=D1=83=20=D1=81=D0=B1=D0=BE=D1=80=D0=BA=D0=B8?= =?UTF-8?q?=20=D0=BE=D0=B1=D1=80=D0=B0=D0=B7=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/ru/kirillius/mktbk/App.java | 125 ++++++++------- .../ru/kirillius/mktbk/ContainerMetadata.java | 21 +++ .../kirillius/mktbk/DockerImageBuilder.java | 150 ++++++++++++++++++ .../ru/kirillius/mktbk/DownloadStatus.java | 5 + src/main/resources/template/config.json | 24 +++ src/main/resources/template/layer.json | 24 +++ src/main/resources/template/manifest.json | 7 + src/main/resources/template/repositories.json | 5 + 8 files changed, 302 insertions(+), 59 deletions(-) create mode 100644 src/main/java/ru/kirillius/mktbk/DockerImageBuilder.java create mode 100644 src/main/resources/template/config.json create mode 100644 src/main/resources/template/layer.json create mode 100644 src/main/resources/template/manifest.json create mode 100644 src/main/resources/template/repositories.json diff --git a/src/main/java/ru/kirillius/mktbk/App.java b/src/main/java/ru/kirillius/mktbk/App.java index c0fe10e..928e68b 100644 --- a/src/main/java/ru/kirillius/mktbk/App.java +++ b/src/main/java/ru/kirillius/mktbk/App.java @@ -110,70 +110,21 @@ public class App { SystemLogger.warning("/!\\ We are going to make a backup. You have to stop all services in this container to prevent data corruption.", LOG_CONTEXT); System.out.print("Type yes to continue or Ctrl+C to abort: "); while (!"yes".equals(cin())) ; - SystemLogger.message("Reading container filesystem metadata...", LOG_CONTEXT); - var filterContents = List.of("/sys/", "/proc/", "/dev/", "/mnt/", "/run/", "/tmp/"); - var files = deviceContext.listFilesystem(container).stream().filter(f -> filterContents.contains(f.getPath()) || filterContents.stream().noneMatch(s -> f.getPath().startsWith(s))).toList(); - SystemLogger.message("Found " + files.size() + " files", LOG_CONTEXT); - - var sftp = deviceContext.getSftpClient(); - final var localTmpFile = "./transfer.tmp"; - - try (var outputStream = new FileOutputStream(container.getGuid())) { - var count = 0; - - try (var tar = new TarArchiveOutputStream(outputStream)) { - tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU); - var status = new DownloadStatus("Downloading", files.size()); - for (var file : files) { - try { - if (file.isDirectory()) { - var entry = new TarArchiveEntry(file.getPath(), TarConstants.LF_DIR); - entry.setGroupId(file.getGroupId()); - entry.setUserId(file.getUserId()); - entry.setMode(file.getMode()); - tar.putArchiveEntry(entry); - tar.closeArchiveEntry(); - status.addCopiedFile(0); - } else if (file.isSymlink()) { - var entry = new TarArchiveEntry(file.getPath(), TarConstants.LF_SYMLINK); - entry.setLinkName(file.getTarget()); - tar.putArchiveEntry(entry); - tar.closeArchiveEntry(); - status.addCopiedFile(0); - } else { - var entry = new TarArchiveEntry(file.getPath()); - entry.setGroupId(file.getGroupId()); - entry.setUserId(file.getUserId()); - entry.setMode(file.getMode()); - var remotePath = "/" + container.getGuid() + file.getPath(); - var size = sftp.size(remotePath); - entry.setSize(size); - tar.putArchiveEntry(entry); + var layerFile = new File(container.getGuid()); + var outputFile = new File("backup_of_" + container.getComment() + ".tar"); - - try { - sftp.get(remotePath, new SFTPFileStream(size, tar)); - }catch (AssertionError ae){ - throw new RuntimeException(ae); - } - - tar.closeArchiveEntry(); - status.addCopiedFile(size); - } - } catch (IOException e) { - status.clear(); - SystemLogger.error("Unable to copy file " + file.getPath(), LOG_CONTEXT, e); - pause(); - } - - } - tar.finish(); - } + if (!layerFile.exists()) { + var files = findFiles(container); + downloadFiles(container, files, layerFile); } - System.out.println(files.size()); + SystemLogger.message("Building docker image " + outputFile.getName(), LOG_CONTEXT); + var builder = new DockerImageBuilder(container); + builder.build(layerFile, outputFile); + SystemLogger.message("Backup is done ", LOG_CONTEXT); + pause(); } finally { if (reader != null) { @@ -189,6 +140,62 @@ public class App { } + private void downloadFiles(ContainerMetadata container, List files, File outputFile) throws IOException { + var sftp = deviceContext.getSftpClient(); + try (var outputStream = new FileOutputStream(outputFile)) { + try (var tar = new TarArchiveOutputStream(outputStream)) { + tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU); + var status = new DownloadStatus("Downloading", files.size()); + for (var file : files) { + try { + if (file.isDirectory()) { + var entry = new TarArchiveEntry(file.getPath(), TarConstants.LF_DIR); + entry.setGroupId(file.getGroupId()); + entry.setUserId(file.getUserId()); + entry.setMode(file.getMode()); + tar.putArchiveEntry(entry); + tar.closeArchiveEntry(); + status.addCopiedFile(0); + } else if (file.isSymlink()) { + var entry = new TarArchiveEntry(file.getPath(), TarConstants.LF_SYMLINK); + entry.setLinkName(file.getTarget()); + tar.putArchiveEntry(entry); + tar.closeArchiveEntry(); + status.addCopiedFile(0); + } else { + var entry = new TarArchiveEntry(file.getPath()); + entry.setGroupId(file.getGroupId()); + entry.setUserId(file.getUserId()); + entry.setMode(file.getMode()); + var remotePath = "/" + container.getGuid() + file.getPath(); + var size = sftp.size(remotePath); + entry.setSize(size); + tar.putArchiveEntry(entry); + sftp.get(remotePath, new SFTPFileStream(size, tar)); + tar.closeArchiveEntry(); + status.addCopiedFile(size); + } + } catch (IOException e) { + status.clear(); + SystemLogger.error("Unable to copy file " + file.getPath(), LOG_CONTEXT, e); + pause(); + } + } + tar.finish(); + status.finish(); + SystemLogger.message("Download of " + files.size() + " has been completed", LOG_CONTEXT); + } + } + } + + private List findFiles(ContainerMetadata container) throws IOException, InterruptedException { + SystemLogger.message("Reading container filesystem metadata...", LOG_CONTEXT); + var filterContents = List.of("/sys/", "/proc/", "/dev/", "/mnt/", "/run/", "/tmp/"); + var files = deviceContext.listFilesystem(container).stream().filter(f -> filterContents.contains(f.getPath()) || filterContents.stream().noneMatch(s -> f.getPath().startsWith(s))).toList(); + SystemLogger.message("Found " + files.size() + " files", LOG_CONTEXT); + return files; + } + // private void openSSH() throws TransportException, ConnectionException { // session = client.startSession(); diff --git a/src/main/java/ru/kirillius/mktbk/ContainerMetadata.java b/src/main/java/ru/kirillius/mktbk/ContainerMetadata.java index 72728f1..f8ba1d0 100644 --- a/src/main/java/ru/kirillius/mktbk/ContainerMetadata.java +++ b/src/main/java/ru/kirillius/mktbk/ContainerMetadata.java @@ -2,6 +2,7 @@ package ru.kirillius.mktbk; import lombok.Getter; import lombok.NoArgsConstructor; +import ru.kirillius.json.JSONProperty; import ru.kirillius.json.JSONSerializable; import java.lang.annotation.ElementType; @@ -72,32 +73,52 @@ public class ContainerMetadata { } + @JSONProperty @FieldAlias(".id") private String id; + @JSONProperty @FieldAlias("name") private String guid; + @JSONProperty private String tag; + @JSONProperty private String comment; + @JSONProperty private String os; + @JSONProperty private String arch; @FieldAlias("interface") + @JSONProperty private String ifname; + @JSONProperty private String entrypoint; + @JSONProperty private String mounts; + @JSONProperty private String dns; + @JSONProperty private String hostname; + @JSONProperty private String workdir; + @JSONProperty @FieldAlias("start-on-boot") private String autostart; + @JSONProperty @FieldAlias("domain-name") private String domainname; + @JSONProperty private String envlist; + @JSONProperty private String logging; + @JSONProperty @FieldAlias("root-dir") private String rootdir; + @JSONProperty @FieldAlias("stop-signal") private String stopsignal; + @JSONProperty private String user; + @JSONProperty private String status; diff --git a/src/main/java/ru/kirillius/mktbk/DockerImageBuilder.java b/src/main/java/ru/kirillius/mktbk/DockerImageBuilder.java new file mode 100644 index 0000000..5b9d085 --- /dev/null +++ b/src/main/java/ru/kirillius/mktbk/DockerImageBuilder.java @@ -0,0 +1,150 @@ +package ru.kirillius.mktbk; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.archivers.tar.TarConstants; +import org.json.JSONArray; +import org.json.JSONObject; +import org.json.JSONTokener; +import ru.kirillius.json.JSONUtility; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.regex.Pattern; + +public class DockerImageBuilder { + private final MessageDigest digest; + private final ContainerMetadata containerMetadata; + + + private JSONObject loadJsonTemplate(String template) { + try (var resource = getClass().getClassLoader().getResourceAsStream("template/" + template + ".json")) { + if (resource == null) { + throw new FileNotFoundException(template + ".json"); + } + return new JSONObject(new JSONTokener(resource)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public DockerImageBuilder(ContainerMetadata containerMetadata) { + try { + digest = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + this.containerMetadata = containerMetadata; + + } + + public void build(File layerFile, File outputFile) throws IOException { + try (var gzip = (new FileOutputStream(outputFile))) { + try (var tar = new TarArchiveOutputStream(gzip)) { + var layer = addLayer(layerFile, tar); + var repos = new JSONObject(); + var imageInfo = containerMetadata.getTag().split(Pattern.quote(":")); + if (imageInfo.length > 1) { + var repo = new JSONObject(); + repo.put(imageInfo[1], layer); + repos.put(imageInfo[0], repo); + } + createStringEntry("repositories", repos.toString(), tar); + var manifest = loadJsonTemplate("manifest"); + var config = loadJsonTemplate("config"); + + config.put("os", containerMetadata.getOs()); + config.put("architecture", containerMetadata.getArch()); + config.getJSONObject("rootfs").getJSONArray("diff_ids").put("sha256:" + layer); + var configHash = hash(config.toString()) + ".json"; + + createStringEntry(configHash, config.toString(), tar); + + manifest.getJSONArray("RepoTags").put(containerMetadata.getTag()); + manifest.getJSONArray("Layers").put(layer + "/layer.tar"); + manifest.put("Config", configHash); + + + var manifsetArray = new JSONArray(); + manifsetArray.put(manifest); + createStringEntry("manifest.json", manifsetArray.toString(), tar); + + tar.finish(); + } + } + + var metafile = new File(outputFile.getPath() + ".meta.json"); + try (var writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(metafile)))) { + writer.write(JSONUtility.serializeStructure(containerMetadata).toString()); + } + } + + private String addLayer(File layer, TarArchiveOutputStream tar) throws IOException { + + var layerHash = hashFile(layer); + createFolder(layerHash, tar); + createFile(layer, layerHash + "/layer.tar", tar); + createStringEntry(layerHash + "/VERSION", "1.0", tar); + var json = loadJsonTemplate("layer"); + json.put("id", layerHash); + json.put("os", containerMetadata.getOs()); + createStringEntry(layerHash + "/json", json.toString(), tar); + return layerHash; + } + + private void createFile(File file, String path, TarArchiveOutputStream tar) throws IOException { + var entry = new TarArchiveEntry(path); + entry.setSize(file.length()); + tar.putArchiveEntry(entry); + Files.copy(file.toPath(), tar); + tar.closeArchiveEntry(); + } + + private void createFolder(String folder, TarArchiveOutputStream tar) throws IOException { + var entry = new TarArchiveEntry(folder, TarConstants.LF_DIR); + tar.putArchiveEntry(entry); + tar.closeArchiveEntry(); + } + + private void createStringEntry(String path, String data, TarArchiveOutputStream tar) throws IOException { + var bytes = data.getBytes(StandardCharsets.UTF_8); + var entry = new TarArchiveEntry(path); + entry.setSize(bytes.length); + tar.putArchiveEntry(entry); + try (var inputStream = new ByteArrayInputStream(bytes)) { + inputStream.transferTo(tar); + } + tar.closeArchiveEntry(); + } + + private String hash(String something) { + return hashToString(digest.digest(something.getBytes(StandardCharsets.UTF_8))); + } + + private String hashFile(File file) throws IOException { + + byte[] buffer = new byte[8192]; + int count; + try (var inputStream = new BufferedInputStream(new FileInputStream(file))) { + while ((count = inputStream.read(buffer)) > 0) { + digest.update(buffer, 0, count); + } + } + return hashToString(digest.digest()); + } + + private String hashToString(byte[] hash) { + var hexString = new StringBuilder(2 * hash.length); + for (byte b : hash) { + var hex = Integer.toHexString(0xff & b); + if (hex.length() == 1) { + hexString.append('0'); + } + hexString.append(hex); + } + return hexString.toString(); + } +} diff --git a/src/main/java/ru/kirillius/mktbk/DownloadStatus.java b/src/main/java/ru/kirillius/mktbk/DownloadStatus.java index 9f1c7ad..48f877f 100644 --- a/src/main/java/ru/kirillius/mktbk/DownloadStatus.java +++ b/src/main/java/ru/kirillius/mktbk/DownloadStatus.java @@ -69,4 +69,9 @@ public class DownloadStatus { } } + public void finish(){ + lastUpdate = 0; + update(); + } + } diff --git a/src/main/resources/template/config.json b/src/main/resources/template/config.json new file mode 100644 index 0000000..4d6b823 --- /dev/null +++ b/src/main/resources/template/config.json @@ -0,0 +1,24 @@ +{ + "architecture": "amd64", + "author": "Container backup utility", + "config": { + "Env": [ + ], + "Cmd": [ + ], + "Labels": { + }, + "ArgsEscaped": true, + "OnBuild": null + }, + "created": "1970-01-01T03:00:00+03:00", + "history": [ + ], + "os": "linux", + "rootfs": { + "type": "layers", + "diff_ids": [ + + ] + } +} \ No newline at end of file diff --git a/src/main/resources/template/layer.json b/src/main/resources/template/layer.json new file mode 100644 index 0000000..eb93d31 --- /dev/null +++ b/src/main/resources/template/layer.json @@ -0,0 +1,24 @@ +{ + "id": "f407f64ec36e5c33a709060448ba6e4f955cd19e60543b84bf059ae6c5a09113", + "created": "1970-01-01T03:00:00+03:00", + "container_config": { + "Hostname": "", + "Domainname": "", + "User": "", + "AttachStdin": false, + "AttachStdout": false, + "AttachStderr": false, + "Tty": false, + "OpenStdin": false, + "StdinOnce": false, + "Env": null, + "Cmd": null, + "Image": "", + "Volumes": null, + "WorkingDir": "", + "Entrypoint": null, + "OnBuild": null, + "Labels": null + }, + "os": "linux" +} \ No newline at end of file diff --git a/src/main/resources/template/manifest.json b/src/main/resources/template/manifest.json new file mode 100644 index 0000000..20b17f9 --- /dev/null +++ b/src/main/resources/template/manifest.json @@ -0,0 +1,7 @@ +{ + "Config": "8550fec822c523351b294d5726da0dd8635879e5ae03c9abd3ae66f85f0ca97a.json", + "RepoTags": [ + ], + "Layers": [ + ] +} diff --git a/src/main/resources/template/repositories.json b/src/main/resources/template/repositories.json new file mode 100644 index 0000000..63ce04e --- /dev/null +++ b/src/main/resources/template/repositories.json @@ -0,0 +1,5 @@ +{ + "opensuse-ssh-supervisord": { + "latest": "a7ab6e7cbb92df484a7302d3ecc084e3e5e45c1f7aac6a1e69110bf196f65b3a" + } +} \ No newline at end of file