kirillius
/
mikrotik-container-backup-utility
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Добавил фичу сборки образа

This commit is contained in:
kirillius 2025-01-30 22:26:14 +03:00
parent 27ffa6ef6d
commit 0fae695941
8 changed files with 302 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
src/main/java/ru/kirillius/mktbk/App.java
View File

@ -110,17 +110,39 @@ public class App {
SystemLogger.warning("/!\\ We are going to make a backup. You have to stop all services in this container to prevent data corruption.", LOG_CONTEXT);
System.out.print("Type yes to continue or Ctrl+C to abort: ");
while (!"yes".equals(cin())) ;
SystemLogger.message("Reading container filesystem metadata...", LOG_CONTEXT);
var filterContents = List.of("/sys/", "/proc/", "/dev/", "/mnt/", "/run/", "/tmp/");
var files = deviceContext.listFilesystem(container).stream().filter(f -> filterContents.contains(f.getPath()) || filterContents.stream().noneMatch(s -> f.getPath().startsWith(s))).toList();
SystemLogger.message("Found " + files.size() + " files", LOG_CONTEXT);
var layerFile = new File(container.getGuid());
var outputFile = new File("backup_of_" + container.getComment() + ".tar");
if (!layerFile.exists()) {
var files = findFiles(container);
downloadFiles(container, files, layerFile);
}
SystemLogger.message("Building docker image " + outputFile.getName(), LOG_CONTEXT);
var builder = new DockerImageBuilder(container);
builder.build(layerFile, outputFile);
SystemLogger.message("Backup is done ", LOG_CONTEXT);
pause();
} finally {
if (reader != null) {
reader.close();
}
if (deviceContext != null) {
deviceContext.close();
}
}
// openSFTP();
}
private void downloadFiles(ContainerMetadata container, List<FileMetadata> files, File outputFile) throws IOException {
var sftp = deviceContext.getSftpClient();
final var localTmpFile = "./transfer.tmp";
try (var outputStream = new FileOutputStream(container.getGuid())) {
var count = 0;
try (var outputStream = new FileOutputStream(outputFile)) {
try (var tar = new TarArchiveOutputStream(outputStream)) {
tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
var status = new DownloadStatus("Downloading", files.size());
@ -147,18 +169,9 @@ public class App {
entry.setMode(file.getMode());
var remotePath = "/" + container.getGuid() + file.getPath();
var size = sftp.size(remotePath);
entry.setSize(size);
tar.putArchiveEntry(entry);
try {
sftp.get(remotePath, new SFTPFileStream(size, tar));
}catch (AssertionError ae){
throw new RuntimeException(ae);
}
tar.closeArchiveEntry();
status.addCopiedFile(size);
}
@ -167,26 +180,20 @@ public class App {
SystemLogger.error("Unable to copy file " + file.getPath(), LOG_CONTEXT, e);
pause();
}
}
tar.finish();
status.finish();
SystemLogger.message("Download of " + files.size() + " has been completed", LOG_CONTEXT);
}
}
}
System.out.println(files.size());
} finally {
if (reader != null) {
reader.close();
}
if (deviceContext != null) {
deviceContext.close();
}
}
// openSFTP();
private List<FileMetadata> findFiles(ContainerMetadata container) throws IOException, InterruptedException {
SystemLogger.message("Reading container filesystem metadata...", LOG_CONTEXT);
var filterContents = List.of("/sys/", "/proc/", "/dev/", "/mnt/", "/run/", "/tmp/");
var files = deviceContext.listFilesystem(container).stream().filter(f -> filterContents.contains(f.getPath()) || filterContents.stream().noneMatch(s -> f.getPath().startsWith(s))).toList();
SystemLogger.message("Found " + files.size() + " files", LOG_CONTEXT);
return files;
}

21
src/main/java/ru/kirillius/mktbk/ContainerMetadata.java
View File

@ -2,6 +2,7 @@ package ru.kirillius.mktbk;
import lombok.Getter;
import lombok.NoArgsConstructor;
import ru.kirillius.json.JSONProperty;
import ru.kirillius.json.JSONSerializable;
import java.lang.annotation.ElementType;
@ -72,32 +73,52 @@ public class ContainerMetadata {
}
@JSONProperty
@FieldAlias(".id")
private String id;
@JSONProperty
@FieldAlias("name")
private String guid;
@JSONProperty
private String tag;
@JSONProperty
private String comment;
@JSONProperty
private String os;
@JSONProperty
private String arch;
@FieldAlias("interface")
@JSONProperty
private String ifname;
@JSONProperty
private String entrypoint;
@JSONProperty
private String mounts;
@JSONProperty
private String dns;
@JSONProperty
private String hostname;
@JSONProperty
private String workdir;
@JSONProperty
@FieldAlias("start-on-boot")
private String autostart;
@JSONProperty
@FieldAlias("domain-name")
private String domainname;
@JSONProperty
private String envlist;
@JSONProperty
private String logging;
@JSONProperty
@FieldAlias("root-dir")
private String rootdir;
@JSONProperty
@FieldAlias("stop-signal")
private String stopsignal;
@JSONProperty
private String user;
@JSONProperty
private String status;

150
src/main/java/ru/kirillius/mktbk/DockerImageBuilder.java Normal file
View File

@ -0,0 +1,150 @@
package ru.kirillius.mktbk;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.archivers.tar.TarConstants;
import org.json.JSONArray;
import org.json.JSONObject;
import org.json.JSONTokener;
import ru.kirillius.json.JSONUtility;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.regex.Pattern;
public class DockerImageBuilder {
private final MessageDigest digest;
private final ContainerMetadata containerMetadata;
private JSONObject loadJsonTemplate(String template) {
try (var resource = getClass().getClassLoader().getResourceAsStream("template/" + template + ".json")) {
if (resource == null) {
throw new FileNotFoundException(template + ".json");
}
return new JSONObject(new JSONTokener(resource));
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public DockerImageBuilder(ContainerMetadata containerMetadata) {
try {
digest = MessageDigest.getInstance("SHA-256");
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
this.containerMetadata = containerMetadata;
}
public void build(File layerFile, File outputFile) throws IOException {
try (var gzip = (new FileOutputStream(outputFile))) {
try (var tar = new TarArchiveOutputStream(gzip)) {
var layer = addLayer(layerFile, tar);
var repos = new JSONObject();
var imageInfo = containerMetadata.getTag().split(Pattern.quote(":"));
if (imageInfo.length > 1) {
var repo = new JSONObject();
repo.put(imageInfo[1], layer);
repos.put(imageInfo[0], repo);
}
createStringEntry("repositories", repos.toString(), tar);
var manifest = loadJsonTemplate("manifest");
var config = loadJsonTemplate("config");
config.put("os", containerMetadata.getOs());
config.put("architecture", containerMetadata.getArch());
config.getJSONObject("rootfs").getJSONArray("diff_ids").put("sha256:" + layer);
var configHash = hash(config.toString()) + ".json";
createStringEntry(configHash, config.toString(), tar);
manifest.getJSONArray("RepoTags").put(containerMetadata.getTag());
manifest.getJSONArray("Layers").put(layer + "/layer.tar");
manifest.put("Config", configHash);
var manifsetArray = new JSONArray();
manifsetArray.put(manifest);
createStringEntry("manifest.json", manifsetArray.toString(), tar);
tar.finish();
}
}
var metafile = new File(outputFile.getPath() + ".meta.json");
try (var writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(metafile)))) {
writer.write(JSONUtility.serializeStructure(containerMetadata).toString());
}
}
private String addLayer(File layer, TarArchiveOutputStream tar) throws IOException {
var layerHash = hashFile(layer);
createFolder(layerHash, tar);
createFile(layer, layerHash + "/layer.tar", tar);
createStringEntry(layerHash + "/VERSION", "1.0", tar);
var json = loadJsonTemplate("layer");
json.put("id", layerHash);
json.put("os", containerMetadata.getOs());
createStringEntry(layerHash + "/json", json.toString(), tar);
return layerHash;
}
private void createFile(File file, String path, TarArchiveOutputStream tar) throws IOException {
var entry = new TarArchiveEntry(path);
entry.setSize(file.length());
tar.putArchiveEntry(entry);
Files.copy(file.toPath(), tar);
tar.closeArchiveEntry();
}
private void createFolder(String folder, TarArchiveOutputStream tar) throws IOException {
var entry = new TarArchiveEntry(folder, TarConstants.LF_DIR);
tar.putArchiveEntry(entry);
tar.closeArchiveEntry();
}
private void createStringEntry(String path, String data, TarArchiveOutputStream tar) throws IOException {
var bytes = data.getBytes(StandardCharsets.UTF_8);
var entry = new TarArchiveEntry(path);
entry.setSize(bytes.length);
tar.putArchiveEntry(entry);
try (var inputStream = new ByteArrayInputStream(bytes)) {
inputStream.transferTo(tar);
}
tar.closeArchiveEntry();
}
private String hash(String something) {
return hashToString(digest.digest(something.getBytes(StandardCharsets.UTF_8)));
}
private String hashFile(File file) throws IOException {
byte[] buffer = new byte[8192];
int count;
try (var inputStream = new BufferedInputStream(new FileInputStream(file))) {
while ((count = inputStream.read(buffer)) > 0) {
digest.update(buffer, 0, count);
}
}
return hashToString(digest.digest());
}
private String hashToString(byte[] hash) {
var hexString = new StringBuilder(2 * hash.length);
for (byte b : hash) {
var hex = Integer.toHexString(0xff & b);
if (hex.length() == 1) {
hexString.append('0');
}
hexString.append(hex);
}
return hexString.toString();
}
}

5
src/main/java/ru/kirillius/mktbk/DownloadStatus.java
View File

@ -69,4 +69,9 @@ public class DownloadStatus {
}
}
public void finish(){
lastUpdate = 0;
update();
}
}

24
src/main/resources/template/config.json Normal file
View File

@ -0,0 +1,24 @@
{
"architecture": "amd64",
"author": "Container backup utility",
"config": {
"Env": [
],
"Cmd": [
],
"Labels": {
},
"ArgsEscaped": true,
"OnBuild": null
},
"created": "1970-01-01T03:00:00+03:00",
"history": [
],
"os": "linux",
"rootfs": {
"type": "layers",
"diff_ids": [
]
}
}

24
src/main/resources/template/layer.json Normal file
View File

@ -0,0 +1,24 @@
{
"id": "f407f64ec36e5c33a709060448ba6e4f955cd19e60543b84bf059ae6c5a09113",
"created": "1970-01-01T03:00:00+03:00",
"container_config": {
"Hostname": "",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": null,
"Cmd": null,
"Image": "",
"Volumes": null,
"WorkingDir": "",
"Entrypoint": null,
"OnBuild": null,
"Labels": null
},
"os": "linux"
}

7
src/main/resources/template/manifest.json Normal file
View File

@ -0,0 +1,7 @@
{
"Config": "8550fec822c523351b294d5726da0dd8635879e5ae03c9abd3ae66f85f0ca97a.json",
"RepoTags": [
],
"Layers": [
]
}

5
src/main/resources/template/repositories.json Normal file
View File

@ -0,0 +1,5 @@
{
"opensuse-ssh-supervisord": {
"latest": "a7ab6e7cbb92df484a7302d3ecc084e3e5e45c1f7aac6a1e69110bf196f65b3a"
}
}