New: Do not store the full data after everything but (block) entities has been loaded

build.gradle

@@ -19,6 +19,7 @@ subprojects {
             "dedupmultipart",
             "blockstatecache",
             "dedupbakedquad",
+            "chunknbt",
         ].stream()
         .map({s -> rootProject.archives_base_name+"."+s+".mixin.json"})
         .collect(Collectors.toList())

common/src/main/java/malte0811/ferritecore/impl/ChunkNBTImpl.java

@@ -0,0 +1,33 @@
+package malte0811.ferritecore.impl;
+
+import net.minecraft.nbt.CompoundTag;
+import net.minecraft.nbt.Tag;
+
+import java.util.Objects;
+
+public class ChunkNBTImpl {
+    private static final ThreadLocal<CompoundTag> TEMP_LEVEL_NBT = new ThreadLocal<>();
+
+    public static void extractNBT(CompoundTag fullTag) {
+        CompoundTag fullLevelNBT = fullTag.getCompound("Level");
+        CompoundTag onlyRelevant = new CompoundTag();
+        copyTagFrom(fullLevelNBT, onlyRelevant, "Entities");
+        copyTagFrom(fullLevelNBT, onlyRelevant, "TileEntities");
+        TEMP_LEVEL_NBT.set(onlyRelevant);
+    }
+
+    private static void copyTagFrom(CompoundTag from, CompoundTag to, String key) {
+        Tag subtag = from.get(key);
+        if (subtag != null) {
+            to.put(key, subtag);
+        }
+    }
+
+    public static CompoundTag getExtractedNBT() {
+        return Objects.requireNonNull(TEMP_LEVEL_NBT.get());
+    }
+
+    public static void clearExtractedNBT() {
+        TEMP_LEVEL_NBT.set(null);
+    }
+}

common/src/main/java/malte0811/ferritecore/mixin/chunknbt/ChunkSerializerMixin.java

@@ -0,0 +1,60 @@
+package malte0811.ferritecore.mixin.chunknbt;
+
+import malte0811.ferritecore.impl.ChunkNBTImpl;
+import net.minecraft.nbt.CompoundTag;
+import net.minecraft.server.level.ServerLevel;
+import net.minecraft.world.entity.ai.village.poi.PoiManager;
+import net.minecraft.world.level.ChunkPos;
+import net.minecraft.world.level.Level;
+import net.minecraft.world.level.TickList;
+import net.minecraft.world.level.block.Block;
+import net.minecraft.world.level.chunk.*;
+import net.minecraft.world.level.chunk.storage.ChunkSerializer;
+import net.minecraft.world.level.levelgen.structure.templatesystem.StructureManager;
+import net.minecraft.world.level.material.Fluid;
+import org.spongepowered.asm.mixin.Mixin;
+import org.spongepowered.asm.mixin.Shadow;
+import org.spongepowered.asm.mixin.injection.At;
+import org.spongepowered.asm.mixin.injection.Inject;
+import org.spongepowered.asm.mixin.injection.ModifyArg;
+import org.spongepowered.asm.mixin.injection.callback.CallbackInfoReturnable;
+
+import java.util.function.Consumer;
+
+@Mixin(ChunkSerializer.class)
+public abstract class ChunkSerializerMixin {
+    @Shadow
+    private static void postLoadChunk(ServerLevel serverLevel, CompoundTag compoundTag, LevelChunk levelChunk) {}
+
+    @Inject(
+            method = "read",
+            at = @At(value = "NEW", target = "net/minecraft/world/level/chunk/LevelChunk")
+    )
+    private static void extractLevelNBT(
+            ServerLevel _1, StructureManager _2, PoiManager _3, ChunkPos _4, CompoundTag fullNBT,
+            CallbackInfoReturnable<ProtoChunk> cir
+    ) {
+        ChunkNBTImpl.extractNBT(fullNBT);
+    }
+
+    @ModifyArg(
+            method = "read",
+            at = @At(value = "INVOKE", target = "Lnet/minecraft/world/level/chunk/LevelChunk;<init>(Lnet/minecraft/world/level/Level;Lnet/minecraft/world/level/ChunkPos;Lnet/minecraft/world/level/chunk/ChunkBiomeContainer;Lnet/minecraft/world/level/chunk/UpgradeData;Lnet/minecraft/world/level/TickList;Lnet/minecraft/world/level/TickList;J[Lnet/minecraft/world/level/chunk/LevelChunkSection;Ljava/util/function/Consumer;)V"),
+            index = 8
+    )
+    private static Consumer<LevelChunk> replacePostLoad(
+            Level level, ChunkPos pos, ChunkBiomeContainer biomes, UpgradeData upgradeData, TickList<Block> blockTicks,
+            TickList<Fluid> fluidTicks, long l, LevelChunkSection[] sections, Consumer<LevelChunk> consumer
+    ) {
+        CompoundTag strippedNBT = ChunkNBTImpl.getExtractedNBT();
+        return levelChunk -> postLoadChunk((ServerLevel) level, strippedNBT, levelChunk);
+    }
+
+    @Inject(method = "read", at = @At("RETURN"))
+    private static void resetExtractedNBT(
+            ServerLevel _1, StructureManager _2, PoiManager _3, ChunkPos _4, CompoundTag _5,
+            CallbackInfoReturnable<ProtoChunk> cir
+    ) {
+        ChunkNBTImpl.clearExtractedNBT();
+    }
+}

common/src/main/java/malte0811/ferritecore/mixin/chunknbt/Config.java

@@ -0,0 +1,10 @@
+package malte0811.ferritecore.mixin.chunknbt;
+
+import malte0811.ferritecore.mixin.config.FerriteConfig;
+import malte0811.ferritecore.mixin.config.FerriteMixinConfig;
+
+public class Config extends FerriteMixinConfig {
+    public Config() {
+        super(FerriteConfig.REDUCED_CHUNK_NBT, true);
+    }
+}

common/src/main/java/malte0811/ferritecore/mixin/config/FerriteConfig.java

@@ -18,6 +18,7 @@ public class FerriteConfig {
     public static final Option DEDUP_QUADS;
     public static final Option COMPACT_FAST_MAP;
     public static final Option POPULATE_NEIGHBOR_TABLE;
+    public static final Option REDUCED_CHUNK_NBT;
 
     static {
         ConfigBuilder builder = new ConfigBuilder();
@@ -50,6 +51,10 @@ public class FerriteConfig {
                 "bakedQuadDeduplication",
                 "Deduplicate vertex data of baked quads in the basic model implementations"
         );
+        REDUCED_CHUNK_NBT = builder.createOption(
+                "reducedChunkNBT",
+                "Do not keep already-parsed NBT data for partially loaded chunks in memory"
+        );
         COMPACT_FAST_MAP = builder.createOptInOption(
                 "compactFastMap",
                 "Use a slightly more compact, but also slightly slower representation for block states"

common/src/main/resources/ferritecore.chunknbt.mixin.json

@@ -0,0 +1,14 @@
+{
+  "required": true,
+  "package": "malte0811.ferritecore.mixin.chunknbt",
+  "compatibilityLevel": "JAVA_16",
+  "client": [],
+  "injectors": {
+    "defaultRequire": 1
+  },
+  "minVersion": "0.8",
+  "plugin": "malte0811.ferritecore.mixin.chunknbt.Config",
+  "mixins": [
+    "ChunkSerializerMixin"
+  ]
+}

fabric/src/main/resources/fabric.mod.json

@@ -31,6 +31,7 @@
     "ferritecore.mrl.mixin.json",
     "ferritecore.predicates.mixin.json",
     "ferritecore.dedupbakedquad.mixin.json",
+    "ferritecore.chunknbt.mixin.json",
     "ferritecore.fabric.mixin.json"
   ]
 }

summary.md

@@ -136,3 +136,19 @@ Saved memory: Close to 150 MB
 CPU impact: Some during model loading, none afterwards  
 Side: client  
 Mixin subpackage: `bakedquad`
+
+### 9. NBT data of partially loaded chunks
+
+Chunks are loaded from NBT in two steps. First everything but entities and block entities is loaded, then those are
+created later in the second step. Minecraft tries to keep a "buffer" of "half-loaded" chunks (i.e. ones that have
+completed the first, but not the second step) around the "fully loaded" chunks (i.e. ones that have completed both
+steps). By default, the full chunk NBT is kept between the two steps, rather than just the sub-tags for entities and
+block entities. Since those typically only account for a small fraction of the chunk data this wastes a decent amount of
+RAM.
+
+Saved memory: 90-100 MB  
+CPU impact: Minimal during chunk loading (creating a new compound tag with just entities and block entites)  
+Side: server  
+Mixin subpackage: `chunknbt`  
+Notes: The memory impact is practically independent of mods, but does depend on the loaded chunk regions. The chunks
+around the fully loaded area need to have been generated, either by a player or using a chunk pregenerator command.