Write chunk NBT vanilla reads: flat root, byte section Y, four-bit block palettes

Our region files were not Anvil. Three things stood between them and vanilla's
deserializer, and each is enough on its own:

  * everything was nested under a "Level" compound. Chunk data lived there until
    1.18; SerializableChunkData builds a flat root and never looks for the key.
  * a section's Y was an Int. Vanilla writes putByte and reads getByteOr, so
    every section of ours decodes as index 0 and overwrites the one before it.
  * block palettes were packed at ceil(log2(size)) bits. Strategy's tableswitch
    routes bit counts 1 through 4 to the same four-bit configuration, so a
    palette of 2..16 states is four bits wide on disk. Ours were one to three,
    which makes the long array a quarter of the length vanilla computes, and
    SimpleBitStorage rejects the section outright rather than misreading it.

Biome containers were already right: Strategy has no such floor for them, and a
Global configuration above three bits still stores palette indices, just at its
own width. The suspicion that biomes collapsed on save/load was unfounded --
what let it stand is that every round-trip test in this package set blocks and
asserted blocks, so nothing proved biomes survived. They do now, per cell, for
every palette width a section can hold.

Verified against vanilla rather than against our reading of it:
tools/VanillaChunkFormatCheck.java opens a region file we wrote using vanilla's
RegionFile, NbtIo, Strategy and SimpleBitStorage. Sixteen generated chunks pass.
Reverting either the Y type or the palette floor makes it fail with vanilla's
own message -- "Invalid length given for storage, got: 64 but expected: 256" --
so the check can fail, which is the only reason to trust it passing.

Reading a world the official server generated is what the surface-height parity
capture in CLAUDE.md has always needed, and this is half of it.

internal/world/anvil_format_test.go

@@ -0,0 +1,168 @@
+package world
+
+import (
+	"fmt"
+	"testing"
+
+	"regionio/internal/nbt"
+)
+
+// TestChunkNBTIsVanillaAnvil pins the three things that made our region files
+// unreadable by the official server, and its files unreadable by us: chunk data
+// nested under a "Level" compound (where it lived until 1.18), a section index
+// written as an Int where vanilla writes and reads a byte, and block palettes
+// packed tighter than vanilla's four-bit floor.
+func TestChunkNBTIsVanillaAnvil(t *testing.T) {
+	c := NewChunk(3, -5, BiomePlains)
+	for lx := 0; lx < 16; lx++ {
+		for lz := 0; lz < 16; lz++ {
+			c.SetBlock(lx, 0, lz, StateStone)
+		}
+	}
+	c.SetBlock(0, 0, 0, StateDirt) // a second palette entry
+
+	root := chunkToNBT(c)
+	if _, ok := root.Get("Level"); ok {
+		t.Error("chunk NBT still nests under Level; vanilla reads a flat root")
+	}
+	for _, key := range []string{"xPos", "yPos", "zPos", "Status", "sections", "Heightmaps", "block_entities"} {
+		if _, ok := root.Get(key); !ok {
+			t.Errorf("chunk NBT root is missing %q", key)
+		}
+	}
+
+	secTag, ok := root.Get("sections")
+	if !ok {
+		t.Fatal("no sections")
+	}
+	sections := secTag.(nbt.List)
+	if len(sections.Elems) != SectionCount {
+		t.Fatalf("%d sections, want %d", len(sections.Elems), SectionCount)
+	}
+	for i, e := range sections.Elems {
+		sec := e.(*nbt.Compound)
+		y, ok := sec.Get("Y")
+		if !ok {
+			t.Fatalf("section %d has no Y", i)
+		}
+		if _, isByte := y.(nbt.Byte); !isByte {
+			t.Fatalf("section %d Y is %T, want nbt.Byte", i, y)
+		}
+		if got, _ := nbtAsSectionY(sec, "Y"); got != i+minYSection {
+			t.Fatalf("section %d Y decodes to %d, want %d", i, got, i+minYSection)
+		}
+	}
+}
+
+// TestPaletteStorageWidths checks the packed long array is the length vanilla
+// computes from the palette size alone. A section packed at the wrong width has
+// the wrong number of longs, and vanilla's SimpleBitStorage rejects it outright
+// rather than reading it crooked.
+func TestPaletteStorageWidths(t *testing.T) {
+	blockCases := []struct{ palette, bits int }{
+		{1, 0}, {2, 4}, {5, 4}, {16, 4}, {17, 5}, {32, 5}, {33, 6}, {64, 6}, {257, 9},
+	}
+	for _, c := range blockCases {
+		if got := blockStorageBits(c.palette); got != c.bits {
+			t.Errorf("blockStorageBits(%d) = %d, want %d", c.palette, got, c.bits)
+		}
+	}
+	biomeCases := []struct{ palette, bits int }{
+		{1, 0}, {2, 1}, {3, 2}, {4, 2}, {5, 3}, {8, 3}, {9, 4}, {16, 4}, {33, 6}, {64, 6},
+	}
+	for _, c := range biomeCases {
+		if got := biomeStorageBits(c.palette); got != c.bits {
+			t.Errorf("biomeStorageBits(%d) = %d, want %d", c.palette, got, c.bits)
+		}
+	}
+
+	// The four-bit floor is the part that used to be wrong: a two-entry block
+	// palette must still occupy 4096 entries at 4 bits, which is 256 longs.
+	c := NewChunk(0, 0, BiomePlains)
+	c.SetBlock(0, 0, 0, StateStone)
+	c.SetBlock(1, 0, 0, StateDirt)
+	sec := sectionToNBT(c, (0-MinY)>>4)
+	bs := mustCompound(t, sec, "block_states")
+	data, ok := bs.Get("data")
+	if !ok {
+		t.Fatal("a two-entry block palette wrote no data array")
+	}
+	if got, want := len(data.(nbt.LongArray)), sectionVol/(64/4); got != want {
+		t.Errorf("two-entry block palette packed into %d longs, want %d (4 bits, 16 per long)", got, want)
+	}
+}
+
+// TestStoreBiomeRoundTrip is the coverage whose absence let a phantom bug stand:
+// every save/load test in this package set blocks and asserted blocks, so
+// nothing proved the biomes survived. They do — this keeps it that way.
+func TestStoreBiomeRoundTrip(t *testing.T) {
+	original := NewChunk(2, -3, BiomePlains)
+	// Three distinct biomes inside one section, so the palette needs two bits
+	// and the packed array is actually exercised.
+	const y = 0
+	original.SetBiome(0, y, 0, 1)
+	original.SetBiome(4, y, 0, 2)
+	original.SetBiome(8, y, 8, 3)
+	// A second section with a different spread, and one cell per 4x4x4 cell in
+	// a third so the palette is wide.
+	for i := 0; i < biomeCellsPerSection; i++ {
+		bx, by, bz := i&3, (i>>4)&3, (i>>2)&3
+		original.SetBiome(bx*4, 32+by*4, bz*4, uint16(i%17))
+	}
+	// A section left untouched keeps the chunk-wide fallback rather than an
+	// array, which is the single-entry-palette branch on both sides.
+	original.biome = 7
+
+	decoded, err := nbtToChunk(chunkToNBT(original), 0, -1, 2, 29)
+	if err != nil {
+		t.Fatalf("round trip: %v", err)
+	}
+	for si := 0; si < SectionCount; si++ {
+		for i := 0; i < biomeCellsPerSection; i++ {
+			bx, by, bz := i&3, (i>>4)&3, (i>>2)&3
+			lx, ly, lz := bx*4, MinY+si*16+by*4, bz*4
+			if got, want := decoded.GetBiome(lx, ly, lz), original.GetBiome(lx, ly, lz); got != want {
+				t.Fatalf("section %d cell %d at (%d,%d,%d): biome %d, want %d", si, i, lx, ly, lz, got, want)
+			}
+		}
+	}
+}
+
+// TestStoreBiomePaletteWidthSweep walks every palette size a section can hold,
+// which is the axis a change to the index packer would break.
+func TestStoreBiomePaletteWidthSweep(t *testing.T) {
+	for distinct := 1; distinct <= biomeCellsPerSection; distinct++ {
+		t.Run(fmt.Sprintf("palette-%d", distinct), func(t *testing.T) {
+			original := NewChunk(0, 0, BiomePlains)
+			const si = 8
+			for i := 0; i < biomeCellsPerSection; i++ {
+				bx, by, bz := i&3, (i>>4)&3, (i>>2)&3
+				original.SetBiome(bx*4, MinY+si*16+by*4, bz*4, uint16(i%distinct))
+			}
+			decoded, err := nbtToChunk(chunkToNBT(original), 0, 0, 0, 0)
+			if err != nil {
+				t.Fatalf("round trip: %v", err)
+			}
+			for i := 0; i < biomeCellsPerSection; i++ {
+				bx, by, bz := i&3, (i>>4)&3, (i>>2)&3
+				lx, ly, lz := bx*4, MinY+si*16+by*4, bz*4
+				if got, want := decoded.GetBiome(lx, ly, lz), original.GetBiome(lx, ly, lz); got != want {
+					t.Fatalf("cell %d: biome %d, want %d", i, got, want)
+				}
+			}
+		})
+	}
+}
+
+func mustCompound(t *testing.T, c *nbt.Compound, key string) *nbt.Compound {
+	t.Helper()
+	tag, ok := c.Get(key)
+	if !ok {
+		t.Fatalf("missing %q", key)
+	}
+	inner, ok := tag.(*nbt.Compound)
+	if !ok {
+		t.Fatalf("%q is %T, want a compound", key, tag)
+	}
+	return inner
+}

internal/world/store.go

@@ -12,8 +12,8 @@ import (
 )
 
 // store.go is the persistence layer between the in-memory Chunk model and the
-// on-disk Anvil region files. It converts a Chunk to/from the "Level"-nested
-// chunk NBT (26.1.2: per-section block_states/biomes, heightmaps, yPos) and
+// on-disk Anvil region files. It converts a Chunk to/from vanilla's chunk NBT
+// (26.1.2: flat root, per-section block_states/biomes, heightmaps, yPos) and
 // routes the compressed NBT through RegionFile.
 //
 // The store keeps one RegionFile per region (32×32 chunks), opened lazily and
@@ -32,7 +32,7 @@ const dataVersion26 = 4790
 // first time it ran: chunkAt prefers the store over the generator, so the
 // already-explored area around spawn keeps its old terrain and every later fix
 // looks like it did nothing in exactly the place you are standing.
-const generatorVersion = 8
+const generatorVersion = 9
 
 // generatorVersionTag is the NBT key holding generatorVersion. It is namespaced
 // because it is ours, not part of the vanilla chunk format.
@@ -284,11 +284,19 @@ func (s *Store) Close() error {
 	return firstErr
 }
 
-// chunkToNBT builds the Level-nested on-disk NBT for a chunk. The wire Encode()
-// format is not reusable here: disk uses named palettes and the 26.1.2 Level
-// layout with per-section biomes.
+// chunkToNBT builds the on-disk NBT for a chunk. The wire Encode() format is
+// not reusable here: disk uses named palettes and per-section biomes.
+//
+// The layout is vanilla Anvil, flat at the root. It used to nest everything
+// under a "Level" compound, which is where chunk data lived until 1.18 and
+// where SerializableChunkData has not looked since — so nothing outside this
+// package could read our region files, and we could not read a world the
+// official server generated. That last part is what the surface-height parity
+// capture needs.
 func chunkToNBT(c *Chunk) *nbt.Compound {
-	level := nbt.NewCompound().
+	root := nbt.NewCompound().
+		Set("DataVersion", nbt.Int(dataVersion26)).
+		Set(generatorVersionTag, nbt.Int(generatorVersion)).
 		Set("xPos", nbt.Int(c.X)).
 		Set("zPos", nbt.Int(c.Z)).
 		Set("yPos", nbt.Int(int32(minYSection))).
@@ -296,7 +304,7 @@ func chunkToNBT(c *Chunk) *nbt.Compound {
 		Set("LastUpdate", nbt.Long(0)).
 		Set("InhabitedTime", nbt.Long(0))
 	if c.lightReady {
-		level.Set("isLightOn", nbt.Byte(1))
+		root.Set("isLightOn", nbt.Byte(1))
 	}
 
 	// Sections: one compound per vertical section, including empty ones so the
@@ -306,17 +314,14 @@ func chunkToNBT(c *Chunk) *nbt.Compound {
 	for si := 0; si < SectionCount; si++ {
 		sections.Elems = append(sections.Elems, sectionToNBT(c, si))
 	}
-	level.Set("sections", sections)
+	root.Set("sections", sections)
 
-	level.Set("Heightmaps", buildHeightmaps(c))
+	root.Set("Heightmaps", buildHeightmaps(c))
 	// Required-but-empty fields so vanilla loads the chunk without complaints.
-	level.Set("block_entities", nbt.List{ElemID: nbt.TagCompound})
-	level.Set("structures", nbt.NewCompound())
+	root.Set("block_entities", nbt.List{ElemID: nbt.TagCompound})
+	root.Set("structures", nbt.NewCompound())
 
-	return nbt.NewCompound().
-		Set("DataVersion", nbt.Int(dataVersion26)).
-		Set(generatorVersionTag, nbt.Int(generatorVersion)).
-		Set("Level", level)
+	return root
 }
 
 // sectionToNBT builds one section compound: Y + block_states + biomes. Palettes
@@ -324,7 +329,9 @@ func chunkToNBT(c *Chunk) *nbt.Compound {
 // reads as "the whole section is this one entry".
 func sectionToNBT(c *Chunk, si int) *nbt.Compound {
 	yIdx := int32(si + minYSection)
-	sec := nbt.NewCompound().Set("Y", nbt.Int(yIdx))
+	// Vanilla writes Y as a byte and reads it with getByteOr; an Int here makes
+	// every section decode as index 0 on the other side.
+	sec := nbt.NewCompound().Set("Y", nbt.Byte(int8(yIdx)))
 
 	// Block states: build a palette of distinct IDs in the section, then a packed
 	// long array of indices (only when more than one distinct value).
@@ -346,8 +353,8 @@ func sectionToNBT(c *Chunk, si int) *nbt.Compound {
 			palList.Elems = append(palList.Elems, blockPaletteEntry(id))
 		}
 		blockStates.Set("palette", palList)
-		if len(palette) > 1 {
-			blockStates.Set("data", packIndices(s[:], indexOf))
+		if bits := blockStorageBits(len(palette)); bits > 0 {
+			blockStates.Set("data", packIndices(s[:], indexOf, bits))
 		}
 	} else {
 		// Empty section → air palette, no data.
@@ -377,8 +384,8 @@ func sectionToNBT(c *Chunk, si int) *nbt.Compound {
 		biomePalList.Elems = append(biomePalList.Elems, nbt.String(biomeNameByID(id)))
 	}
 	biomes.Set("palette", biomePalList)
-	if c.biomes[si] != nil && len(biomePalette) > 1 {
-		biomes.Set("data", packIndices(c.biomes[si][:], biomeIndexOf))
+	if bits := biomeStorageBits(len(biomePalette)); c.biomes[si] != nil && bits > 0 {
+		biomes.Set("data", packIndices(c.biomes[si][:], biomeIndexOf, bits))
 	}
 	sec.Set("biomes", biomes)
 	if c.lightReady {
@@ -434,18 +441,33 @@ func topNonAirY(c *Chunk, x, z int) int {
 	return MinY - 1
 }
 
-// packIndices packs a slice of IDs into a long array using the minimum bit width
-// for the palette size, mirroring the network paletted-container packing (no
-// value spans a long boundary in vanilla's chunk NBT).
-func packIndices(ids []uint16, indexOf map[uint16]int) nbt.LongArray {
-	bits := bitsFor(len(indexOf))
+// blockStorageBits is Strategy$1.getConfigurationForPaletteSize(...).bitsInStorage()
+// for a block palette: nothing at all for a single entry, and never fewer than
+// four bits otherwise. Vanilla's tableswitch sends bit counts 1 through 4 all to
+// the same four-bit linear configuration, so a palette of 2..16 states is stored
+// four bits wide even though two would fit. Packing it tighter, as we did,
+// produces a long array of the wrong length and vanilla refuses the section.
+func blockStorageBits(paletteSize int) int {
+	bits := bitsFor(paletteSize)
+	if bits > 0 && bits < 4 {
+		return 4
+	}
+	return bits
+}
+
+// biomeStorageBits is the same for a biome palette, where Strategy$2 has no
+// floor: the width really is ceil(log2(size)), and a Global configuration above
+// three bits still stores palette indices, just at its own width.
+func biomeStorageBits(paletteSize int) int { return bitsFor(paletteSize) }
+
+// packIndices packs a slice of IDs into a long array at the given bit width,
+// with no value spanning a long boundary — vanilla's SimpleBitStorage layout.
+// A width of zero means the container carries no data array at all.
+func packIndices(ids []uint16, indexOf map[uint16]int, bits int) nbt.LongArray {
 	if bits < 1 {
-		bits = 1
+		return nil
 	}
 	perLong := 64 / bits
-	if perLong == 0 {
-		perLong = 1
-	}
 	numLongs := (len(ids) + perLong - 1) / perLong
 	longs := make(nbt.LongArray, numLongs)
 	for i, id := range ids {
@@ -471,16 +493,8 @@ func nbtToChunk(root *nbt.Compound, regionX, regionZ, localX, localZ int) (*Chun
 		return nil, ErrChunkNotFound
 	}
 
-	levelTag, ok := root.Get("Level")
-	if !ok {
-		return nil, fmt.Errorf("world: chunk NBT missing Level")
-	}
-	level, ok := levelTag.(*nbt.Compound)
-	if !ok {
-		return nil, fmt.Errorf("world: Level is not a compound")
-	}
-	cx := int32(nbtAsInt(level, "xPos"))
-	cz := int32(nbtAsInt(level, "zPos"))
+	cx := int32(nbtAsInt(root, "xPos"))
+	cz := int32(nbtAsInt(root, "zPos"))
 	wantX := int32(regionX*32 + localX)
 	wantZ := int32(regionZ*32 + localZ)
 	if cx != wantX || cz != wantZ {
@@ -488,21 +502,24 @@ func nbtToChunk(root *nbt.Compound, regionX, regionZ, localX, localZ int) (*Chun
 	}
 
 	c := &Chunk{X: cx, Z: cz, biome: BiomePlains}
-	if lightTag, ok := level.Get("isLightOn"); ok {
+	if lightTag, ok := root.Get("isLightOn"); ok {
 		if enabled, ok := lightTag.(nbt.Byte); ok && enabled != 0 {
 			c.lightReady = true
 		}
 	}
 
 	// Sections.
-	if secTag, ok := level.Get("sections"); ok {
+	if secTag, ok := root.Get("sections"); ok {
 		if secList, ok := secTag.(nbt.List); ok && secList.ElemID == nbt.TagCompound {
 			for _, st := range secList.Elems {
 				sc, ok := st.(*nbt.Compound)
 				if !ok {
 					continue
 				}
-				yIdx := int(nbtAsInt(sc, "Y"))
+				yIdx, ok := nbtAsSectionY(sc, "Y")
+				if !ok {
+					continue
+				}
 				si := yIdx - minYSection
 				if si < 0 || si >= SectionCount {
 					continue
@@ -581,7 +598,7 @@ func readBlockStates(c *Chunk, si int, sc *nbt.Compound) {
 	}
 	if dataTag, ok := bs.Get("data"); ok {
 		if data, ok := dataTag.(nbt.LongArray); ok {
-			unpackIndices(s[:], ids, data)
+			unpackIndices(s[:], ids, data, blockStorageBits(len(ids)))
 		}
 	}
 }
@@ -619,7 +636,7 @@ func readBiomes(c *Chunk, si int, sc *nbt.Compound) {
 	if dataTag, ok := bc.Get("data"); ok {
 		if data, ok := dataTag.(nbt.LongArray); ok {
 			cells := new([biomeCellsPerSection]uint16)
-			unpackIndices(cells[:], ids, data)
+			unpackIndices(cells[:], ids, data, biomeStorageBits(len(ids)))
 			c.biomes[si] = cells
 		}
 	}
@@ -653,6 +670,26 @@ func nbtAsInt(c *nbt.Compound, name string) int32 {
 	return 0
 }
 
+// nbtAsSectionY reads a section index, which vanilla writes as a byte. It also
+// accepts a short or an int so a chunk written before we matched vanilla still
+// decodes, and reports whether the tag was there at all — a section with no Y
+// is not section 0, it is malformed.
+func nbtAsSectionY(c *nbt.Compound, name string) (int, bool) {
+	t, ok := c.Get(name)
+	if !ok {
+		return 0, false
+	}
+	switch v := t.(type) {
+	case nbt.Byte:
+		return int(int8(v)), true
+	case nbt.Short:
+		return int(int16(v)), true
+	case nbt.Int:
+		return int(int32(v)), true
+	}
+	return 0, false
+}
+
 func nbtAsString(c *nbt.Compound, name string) nbt.String {
 	if t, ok := c.Get(name); ok {
 		if v, ok := t.(nbt.String); ok {
@@ -664,15 +701,11 @@ func nbtAsString(c *nbt.Compound, name string) nbt.String {
 
 // unpackIndices reverses packIndices: fills dst with palette IDs using the
 // packed long array.
-func unpackIndices(dst []uint16, ids []uint16, data nbt.LongArray) {
-	bits := bitsFor(len(ids))
+func unpackIndices(dst []uint16, ids []uint16, data nbt.LongArray, bits int) {
 	if bits < 1 {
-		bits = 1
+		return
 	}
 	perLong := 64 / bits
-	if perLong == 0 {
-		perLong = 1
-	}
 	mask := int64(1)<<uint(bits) - 1
 	for i := range dst {
 		longIdx := i / perLong