From 5c95693cd22ea5323f4a1b5bd160edbba0807a93 Mon Sep 17 00:00:00 2001 From: Alvinn8 <42838560+Alvinn8@users.noreply.github.com> Date: Sun, 27 Oct 2024 22:26:04 +0100 Subject: [PATCH] WIP Handle world reading in one place and abstract WorldProviders to only reading files --- .../main/java/ca/bkaw/mch/world/FileInfo.java | 24 +++ .../java/ca/bkaw/mch/world/WorldProvider.java | 10 ++ .../java/ca/bkaw/mch/world/WorldReader.java | 142 ++++++++++++++++++ 3 files changed, 176 insertions(+) create mode 100644 mch/src/main/java/ca/bkaw/mch/world/FileInfo.java create mode 100644 mch/src/main/java/ca/bkaw/mch/world/WorldReader.java diff --git a/mch/src/main/java/ca/bkaw/mch/world/FileInfo.java b/mch/src/main/java/ca/bkaw/mch/world/FileInfo.java new file mode 100644 index 0000000..f281f93 --- /dev/null +++ b/mch/src/main/java/ca/bkaw/mch/world/FileInfo.java @@ -0,0 +1,24 @@ +package ca.bkaw.mch.world; + +import ca.bkaw.mch.util.StringPath; +import org.jetbrains.annotations.Nullable; + +/** + * A file name with optional file metadata. + * + * @param name The name of the file or directory. + * @param path The path to the file relative to the provider's root. + * @param metadata Metadata about the file or directory. + */ +public record FileInfo(String name, StringPath path, @Nullable Metadata metadata) { + /** + * Metadata about a file or directory. + * + * @param isFile {@code true} if a regular file. + * @param isDirectory {@code true} if a directory. + * @param fileSize The file size of the file. May be undefined for directories. + * @param lastModified The last modification time of the file, measured in epoch milliseconds. + */ + public record Metadata(boolean isFile, boolean isDirectory, long fileSize, long lastModified) { + } +} diff --git a/mch/src/main/java/ca/bkaw/mch/world/WorldProvider.java b/mch/src/main/java/ca/bkaw/mch/world/WorldProvider.java index 7db17a1..069db4c 100644 --- a/mch/src/main/java/ca/bkaw/mch/world/WorldProvider.java +++ b/mch/src/main/java/ca/bkaw/mch/world/WorldProvider.java @@ -5,6 +5,7 @@ import ca.bkaw.mch.object.tree.Tree; import ca.bkaw.mch.repository.MchRepository; import ca.bkaw.mch.util.RandomAccessReader; +import ca.bkaw.mch.util.StringPath; import org.jetbrains.annotations.Nullable; import java.io.IOException; @@ -19,6 +20,15 @@ * provider object should be closed. */ public interface WorldProvider extends AutoCloseable { + + List list(StringPath path); + @Nullable + FileInfo.Metadata stat(StringPath path); + RandomAccessReader openFile(StringPath path, long estimatedSize); + byte[] readFile(StringPath path, long estimatedSize); + + // Old + /** * Get the dimensions that this world has. * diff --git a/mch/src/main/java/ca/bkaw/mch/world/WorldReader.java b/mch/src/main/java/ca/bkaw/mch/world/WorldReader.java new file mode 100644 index 0000000..767ddb4 --- /dev/null +++ b/mch/src/main/java/ca/bkaw/mch/world/WorldReader.java @@ -0,0 +1,142 @@ +package ca.bkaw.mch.world; + +import ca.bkaw.mch.object.ObjectStorageTypes; +import ca.bkaw.mch.object.Reference20; +import ca.bkaw.mch.object.blob.Blob; +import ca.bkaw.mch.object.dimension.Dimension; +import ca.bkaw.mch.object.tree.Tree; +import ca.bkaw.mch.repository.MchRepository; +import ca.bkaw.mch.util.RandomAccessReader; +import ca.bkaw.mch.util.StringPath; +import ca.bkaw.mch.util.Util; +import ca.bkaw.mch.world.sftp.OutputStreamFileDest; +import net.schmizz.sshj.sftp.RemoteResourceInfo; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.function.Predicate; + +public class WorldReader { + private final WorldProvider provider; + + public WorldReader(WorldProvider provider) { + this.provider = provider; + } + + @NotNull + private FileInfo.Metadata metadata(FileInfo fileInfo) { + if (fileInfo.metadata() != null) { + return fileInfo.metadata(); + } + // Some providers do not include metadata when listing directories. We need + // to stat to get the metadata. We know the file exists, so it's safe to + // assume not null. + return Objects.requireNonNull( + this.provider.stat(fileInfo.path()), + "File not found. Was the file deleted?" + ); + } + + public List getDimensions() { + List directories = this.provider.list(StringPath.root()); + + List dimensions = new ArrayList<>(3); + for (FileInfo fileInfo : directories) { + String dimension = switch (fileInfo.name()) { + case "region" -> Dimension.OVERWORLD; + case Util.NETHER_FOLDER -> Dimension.NETHER; + case Util.THE_END_FOLDER -> Dimension.THE_END; + default -> null; + }; + if (dimension != null && metadata(fileInfo).isDirectory()) { + dimensions.add(dimension); + } + } + // TODO custom dimensions + return dimensions; + } + + private StringPath getDimensionPath(String dimension) { + return switch (dimension) { + case Dimension.OVERWORLD -> StringPath.root(); + case Dimension.NETHER -> StringPath.of(Util.NETHER_FOLDER); + case Dimension.THE_END -> StringPath.of(Util.THE_END_FOLDER); + default -> StringPath.of( "dimensions/" + dimension.replace(':', '/')); + }; + } + + public List getRegionFiles(String dimension) throws IOException { + StringPath path = getDimensionPath(dimension).resolve("region"); + if (this.provider.stat(path) == null) { + return List.of(); + } + List regionFiles = new ArrayList<>(); + + for (FileInfo fileInfo : this.provider.list(path)) { + String fileName = fileInfo.name(); + if (!fileName.startsWith("r.") || !fileName.endsWith(".mca")) { + continue; + } + FileInfo.Metadata metadata = metadata(fileInfo); + + regionFiles.add(new RegionFileInfo( + fileName, metadata.lastModified(), metadata.fileSize() + )); + } + return regionFiles; + } + + public RandomAccessReader openRegionFile(String dimension, String regionFileName, long estimatedSize) throws IOException { + StringPath path = this.getDimensionPath(dimension).resolve("region").resolve(regionFileName); + return this.provider.openFile(path, estimatedSize); + } + + public Reference20 trackDirectoryTree(String dimension, MchRepository repository, Predicate predicate, @Nullable Tree currentTree) throws IOException { + return this.trackDirectoryTreePath(this.getDimensionPath(dimension), repository, predicate, currentTree); + } + + public Reference20 trackDirectoryTreePath(StringPath path, MchRepository repository, Predicate predicate, @Nullable Tree currentTree) throws IOException { + Tree tree = new Tree(); + for (FileInfo file : this.provider.list(path)) { + String name = file.name(); + if (!predicate.test(name)) { + continue; + } + // TODO repository-wide "mchignore" + if (name.contains("ledger.sqlite")) { + continue; + } + FileInfo.Metadata metadata = metadata(file); + if (metadata.isDirectory()) { + // Track subdirectories + Reference20 currentSubTreeReference = currentTree != null ? currentTree.getSubTrees().get(name) : null; + Tree currentSubTree = currentSubTreeReference != null ? currentSubTreeReference.resolve(repository) : null; + Reference20 subDirectoryReference = trackDirectoryTreePath(file.path(), repository, str -> true, currentSubTree); + tree.addSubTree(name, subDirectoryReference); + } else if (metadata.isFile()) { + // Track files + Tree.BlobReference currentBlobReference = currentTree != null ? currentTree.getFiles().get(name) : null; + long lastModified = metadata.lastModified(); + if (currentBlobReference == null || currentBlobReference.lastModified() != lastModified) { + // The file has changed since last commit. Save it anew. + byte[] bytes = this.provider.readFile(file.path(), metadata.fileSize()); + Blob blob = new Blob(bytes); + Reference20 blobReference = ObjectStorageTypes.BLOB.save(blob, repository); + tree.addFile(name, new Tree.BlobReference(blobReference, lastModified)); + } else { + // The file has not changed since last commit. Reuse the reference. + tree.addFile(name, currentBlobReference); + } + } + } + + // Save the tree + return ObjectStorageTypes.TREE.save(tree, repository); + } + +}