Go + Dwarf Fortress + weird data formats = ?

ben_lubar

package df2014

import (
	"bytes"
	"compress/zlib"
	"encoding/binary"
	"fmt"
	"io"
)

// Ported from http://dwarffortresswiki.org/index.php/User:Quietust/wtf23a.php

type wtf23a struct {
	State [624]uint32
	Index uint32
}

func (wtf *wtf23a) init(r io.Reader) error {
	err := binary.Read(r, binary.LittleEndian, wtf)
	if err != nil {
		return err
	}

	if wtf.Index%4 != 0 {
		return fmt.Errorf("df2014: cannot guess compression type")
	}

	wtf.Index /= 4

	if wtf.Index >= 624 {
		return fmt.Errorf("df2014: cannot guess compression type")
	}

	return nil
}

func (wtf *wtf23a) next(mod uint32) uint32 {
	if wtf.Index == 624 {
		for i := range wtf.State {
			y := (wtf.State[i] & 0x80000000) | (wtf.State[(i+1)%624] & 0x7FFFFFFF)
			wtf.State[i] = wtf.State[(i+397)%624] ^ ((y >> 1) & 0x7FFFFFFF)
			if y&1 != 0 {
				wtf.State[i] ^= 0x9908b0df
			}
		}
		wtf.Index = 0
	}
	n := wtf.State[wtf.Index]
	wtf.Index++
	return n % mod
}

type wtf23aReader struct {
	r   io.Reader
	buf []byte
}

func (r *wtf23aReader) Read(p []byte) (n int, err error) {
	if len(r.buf) == 0 {
		err = r.fill()
		if err != nil {
			return
		}
	}

	n = copy(p, r.buf)
	r.buf = r.buf[n:]
	return
}

func (r *wtf23aReader) fill() (err error) {
	var initial wtf23a
	err = initial.init(r.r)
	if err != nil {
		return
	}

	defer func() {
		if err == io.EOF {
			err = io.ErrUnexpectedEOF
		}
	}()

	var length int32
	err = binary.Read(r.r, binary.LittleEndian, &length)
	if err != nil {
		return
	}
	if length < 0 {
		return fmt.Errorf("df2014: negative length (%d)", length)
	}

	var skip [1]byte
	data := make([]byte, length)

	// read data, skipping dummy bytes
	rng := initial
	for i := length - 1; i >= 0; i-- {
		_, err = r.r.Read(data[i : i+1])
		if err != nil {
			return
		}

		if rng.next(2) == 1 {
			rng.next(256)
			_, err = r.r.Read(skip[:])
			if err != nil {
				return
			}
		}
	}

	// decode data
	rng = initial
	for i := length - 1; i >= 0; i-- {
		data[i] -= byte(rng.next(10))
	}

	z, err := zlib.NewReader(bytes.NewReader(data))
	if err != nil {
		return
	}
	defer z.Close()

	var buf bytes.Buffer
	_, err = io.Copy(&buf, z)
	if err != nil {
		return
	}

	r.buf = buf.Bytes()

	return
}

I feel dirty after writing this.

Monarch

@ben_lubar said:

I feel dirty after writing this.

I feel dirty after reading this.

blakeyrat

Is this the most Ben L thread ever?

I think yes.

HardwareGeek

@blakeyrat said:

Is this the most Ben L thread ever?

No, it needs more Lojban. (Not really; nothing needs ~~more~~any Lojban.)

ben_lubar

mlatu

blakeyrat

It could also use a 1400 pixel-tall image, of which maybe 300 of those pixels is actually relevant. But this is pretty good.

Medinoc

Is that... a PRNG's seed used as a cryptographic key to read compressed data? Wow.

boomzilla

@ben_lubar said:

mlatu

...barada....necktie!

mikeTheLiar

Maybe not every little syllable, but basically yeah, I said 'em!

hungrier

I don't really know anything about Go, but I feel there should be a comment at the end that says

//The Aristocrats

Quietust

If you're looking to decode save files from modern versions of Dwarf Fortress, stay away from "wtf23a.php" - that's specifically for old versions of Dwarf Fortress (namely, versions 0.23.130.23a and earlier) and does not apply to any version released in the past 7 years. For modern versions, you might find "rawextract.php" to be more useful.

ben_lubar

I'm actually supporting all four different encodings.

blakeyrat

Playing Dwarf Fortress wasn't geeky enough, I must write a save game editor for Dwarf Fortress!!!

ben_lubar

Yeah, pretty much.

blakeyrat

Maybe you should head to Skyrim Nexus and install some of the sexy naked elf mods, might do you some good.

ben_lubar

Dwarf Fortress has elves, and since they don't die of old age, they probably have the clothes they're wearing rot if they don't have new ones near them.

HardwareGeek

@ben_lubar said:

Yeah, pretty much.

Not only do you have no life, I don't think you deserve one, and I'm pretty sure you wouldn't know what to do with it if you did have one.

Quietust

Technically, there are 3 different formats to detect:

0.31 and later - DWORD version, DWORD compressed, followed by compressed blocks of data (DWORD length, then a BYTE array of that length). Detecting this should be straightforward - the first DWORD is within 1284-1600 (or whatever the latest version is), and the second DWORD is either 0x00000000 or 0x00000001 (if it's 0, there's nothing to do)
0.27 and 0.28 - plain blocks of compressed data (same format as above - the version number is inside the compressed data). The first DWORD should generally be less than 20000, and it should be followed by the bytes 0x78 0xDA.
0.19 thru 0.23 - scrambled blocks of compressed data (as above), where each block is preceded by a 2500-byte Mersenne Twister state (624 DWORDs of data, 1 DWORD for index) which is used to unscramble the compressed data (but NOT the length itself). Detecting this is a bit tricky - the 625th DWORD should be between 0 and 2496 and divisible by 4, and the 626th DWORD should be less than 20000.

ben_lubar

Would you say I'm OPPOSED_TO_LIFE?

ben_lubar

My code does that, but with 10000 instead of 20000. It assumes that the first DWORD of the 23a state will always be ≥ 10000.

Quietust

@ben_lubar said:

My code does that, but with 10000 instead of 20000. It assumes that the first DWORD of the 23a state will always be ≥ 10000.

That means it'll trip up on savegames produced with the "Partially disable save file encryption" binary patch located [url=http://dwarffortresswiki.org/index.php/User:Quietust#Improvements]here[/url], since in that case the first DWORD will be zero. In theory, regular saves could be misdetected as well, since the first DWORD can be any value at all.

blakeyrat

@ben_lubar said:

Dwarf Fortress has elves, and since they don't die of old age, they probably have the clothes they're wearing rot if they don't have new ones near them.

Sexy.

Call me a traditionalist, though, but I think Gamebryo/Creation Engine renders them better.

ben_lubar

How about this:

If the first and second DWORD are both 0, assume it's the 23a save format.
If the second DWORD is 0 or 1, assume it's the newest type of save format.
If the first DWORD is less than 10000 and the next two bytes are 0x78 0xDA, assume it's the 40d save format.
Assume it's the 23a save format.

FrostCat

@ben_lubar said:

mlatu

borada niktu?

Edit: hanzo'ed