gen.go

// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

package main

import (
	"bytes"
	"io/ioutil"
	"log"
	"strings"
	"text/template"
)

const (
	copyright = "" +
		"// Copyright 2016 The Go Authors. All rights reserved.\n" +
		"// Use of this source code is governed by a BSD-style\n" +
		"// license that can be found in the LICENSE file.\n"

	doNotEdit = "// generated by go run gen.go; DO NOT EDIT\n"

	dashDashDash = "// --------"
)

func main() {
	tmpl, err := ioutil.ReadFile("gen_acc_amd64.s.tmpl")
	if err != nil {
		log.Fatalf("ReadFile: %v", err)
	}
	if !bytes.HasPrefix(tmpl, []byte(copyright)) {
		log.Fatal("source template did not start with the copyright header")
	}
	tmpl = tmpl[len(copyright):]

	preamble := []byte(nil)
	if i := bytes.Index(tmpl, []byte(dashDashDash)); i < 0 {
		log.Fatalf("source template did not contain %q", dashDashDash)
	} else {
		preamble, tmpl = tmpl[:i], tmpl[i:]
	}

	t, err := template.New("").Parse(string(tmpl))
	if err != nil {
		log.Fatalf("Parse: %v", err)
	}

	out := bytes.NewBuffer(nil)
	out.WriteString(doNotEdit)
	out.Write(preamble)

	for i, v := range instances {
		if i != 0 {
			out.WriteString("\n")
		}
		if strings.Contains(v.LoadArgs, "{{.ShortName}}") {
			v.LoadArgs = strings.Replace(v.LoadArgs, "{{.ShortName}}", v.ShortName, -1)
		}
		if err := t.Execute(out, v); err != nil {
			log.Fatalf("Execute(%q): %v", v.ShortName, err)
		}
	}

	if err := ioutil.WriteFile("acc_amd64.s", out.Bytes(), 0666); err != nil {
		log.Fatalf("WriteFile: %v", err)
	}
}

var instances = []struct {
	LongName       string
	ShortName      string
	FrameSize      string
	ArgsSize       string
	Args           string
	DstElemSize1   int
	DstElemSize4   int
	XMM3           string
	XMM4           string
	XMM5           string
	XMM6           string
	XMM8           string
	XMM9           string
	XMM10          string
	LoadArgs       string
	Setup          string
	LoadXMMRegs    string
	Add            string
	ClampAndScale  string
	ConvertToInt32 string
	Store4         string
	Store1         string
}{{
	LongName:       "fixedAccumulateOpOver",
	ShortName:      "fxAccOpOver",
	FrameSize:      fxFrameSize,
	ArgsSize:       twoArgArgsSize,
	Args:           "dst []uint8, src []uint32",
	DstElemSize1:   1 * sizeOfUint8,
	DstElemSize4:   4 * sizeOfUint8,
	XMM3:           fxXMM3,
	XMM4:           fxXMM4,
	XMM5:           fxXMM5,
	XMM6:           opOverXMM6,
	XMM8:           opOverXMM8,
	XMM9:           opOverXMM9,
	XMM10:          opOverXMM10,
	LoadArgs:       twoArgLoadArgs,
	Setup:          fxSetup,
	LoadXMMRegs:    fxLoadXMMRegs + "\n" + opOverLoadXMMRegs,
	Add:            fxAdd,
	ClampAndScale:  fxClampAndScale,
	ConvertToInt32: fxConvertToInt32,
	Store4:         opOverStore4,
	Store1:         opOverStore1,
}, {
	LongName:       "fixedAccumulateOpSrc",
	ShortName:      "fxAccOpSrc",
	FrameSize:      fxFrameSize,
	ArgsSize:       twoArgArgsSize,
	Args:           "dst []uint8, src []uint32",
	DstElemSize1:   1 * sizeOfUint8,
	DstElemSize4:   4 * sizeOfUint8,
	XMM3:           fxXMM3,
	XMM4:           fxXMM4,
	XMM5:           fxXMM5,
	XMM6:           opSrcXMM6,
	XMM8:           opSrcXMM8,
	XMM9:           opSrcXMM9,
	XMM10:          opSrcXMM10,
	LoadArgs:       twoArgLoadArgs,
	Setup:          fxSetup,
	LoadXMMRegs:    fxLoadXMMRegs + "\n" + opSrcLoadXMMRegs,
	Add:            fxAdd,
	ClampAndScale:  fxClampAndScale,
	ConvertToInt32: fxConvertToInt32,
	Store4:         opSrcStore4,
	Store1:         opSrcStore1,
}, {
	LongName:       "fixedAccumulateMask",
	ShortName:      "fxAccMask",
	FrameSize:      fxFrameSize,
	ArgsSize:       oneArgArgsSize,
	Args:           "buf []uint32",
	DstElemSize1:   1 * sizeOfUint32,
	DstElemSize4:   4 * sizeOfUint32,
	XMM3:           fxXMM3,
	XMM4:           fxXMM4,
	XMM5:           fxXMM5,
	XMM6:           maskXMM6,
	XMM8:           maskXMM8,
	XMM9:           maskXMM9,
	XMM10:          maskXMM10,
	LoadArgs:       oneArgLoadArgs,
	Setup:          fxSetup,
	LoadXMMRegs:    fxLoadXMMRegs + "\n" + maskLoadXMMRegs,
	Add:            fxAdd,
	ClampAndScale:  fxClampAndScale,
	ConvertToInt32: fxConvertToInt32,
	Store4:         maskStore4,
	Store1:         maskStore1,
}, {
	LongName:       "floatingAccumulateOpOver",
	ShortName:      "flAccOpOver",
	FrameSize:      flFrameSize,
	ArgsSize:       twoArgArgsSize,
	Args:           "dst []uint8, src []float32",
	DstElemSize1:   1 * sizeOfUint8,
	DstElemSize4:   4 * sizeOfUint8,
	XMM3:           flXMM3,
	XMM4:           flXMM4,
	XMM5:           flXMM5,
	XMM6:           opOverXMM6,
	XMM8:           opOverXMM8,
	XMM9:           opOverXMM9,
	XMM10:          opOverXMM10,
	LoadArgs:       twoArgLoadArgs,
	Setup:          flSetup,
	LoadXMMRegs:    flLoadXMMRegs + "\n" + opOverLoadXMMRegs,
	Add:            flAdd,
	ClampAndScale:  flClampAndScale,
	ConvertToInt32: flConvertToInt32,
	Store4:         opOverStore4,
	Store1:         opOverStore1,
}, {
	LongName:       "floatingAccumulateOpSrc",
	ShortName:      "flAccOpSrc",
	FrameSize:      flFrameSize,
	ArgsSize:       twoArgArgsSize,
	Args:           "dst []uint8, src []float32",
	DstElemSize1:   1 * sizeOfUint8,
	DstElemSize4:   4 * sizeOfUint8,
	XMM3:           flXMM3,
	XMM4:           flXMM4,
	XMM5:           flXMM5,
	XMM6:           opSrcXMM6,
	XMM8:           opSrcXMM8,
	XMM9:           opSrcXMM9,
	XMM10:          opSrcXMM10,
	LoadArgs:       twoArgLoadArgs,
	Setup:          flSetup,
	LoadXMMRegs:    flLoadXMMRegs + "\n" + opSrcLoadXMMRegs,
	Add:            flAdd,
	ClampAndScale:  flClampAndScale,
	ConvertToInt32: flConvertToInt32,
	Store4:         opSrcStore4,
	Store1:         opSrcStore1,
}, {
	LongName:       "floatingAccumulateMask",
	ShortName:      "flAccMask",
	FrameSize:      flFrameSize,
	ArgsSize:       twoArgArgsSize,
	Args:           "dst []uint32, src []float32",
	DstElemSize1:   1 * sizeOfUint32,
	DstElemSize4:   4 * sizeOfUint32,
	XMM3:           flXMM3,
	XMM4:           flXMM4,
	XMM5:           flXMM5,
	XMM6:           maskXMM6,
	XMM8:           maskXMM8,
	XMM9:           maskXMM9,
	XMM10:          maskXMM10,
	LoadArgs:       twoArgLoadArgs,
	Setup:          flSetup,
	LoadXMMRegs:    flLoadXMMRegs + "\n" + maskLoadXMMRegs,
	Add:            flAdd,
	ClampAndScale:  flClampAndScale,
	ConvertToInt32: flConvertToInt32,
	Store4:         maskStore4,
	Store1:         maskStore1,
}}

const (
	fxFrameSize = `0`
	flFrameSize = `8`

	oneArgArgsSize = `24`
	twoArgArgsSize = `48`

	sizeOfUint8  = 1
	sizeOfUint32 = 4

	fxXMM3 = `-`
	flXMM3 = `flSignMask`

	fxXMM4 = `-`
	flXMM4 = `flOne`

	fxXMM5 = `fxAlmost65536`
	flXMM5 = `flAlmost65536`

	oneArgLoadArgs = `
		MOVQ buf_base+0(FP), DI
		MOVQ buf_len+8(FP), BX
		MOVQ buf_base+0(FP), SI
		MOVQ buf_len+8(FP), R10
		`
	twoArgLoadArgs = `
		MOVQ dst_base+0(FP), DI
		MOVQ dst_len+8(FP), BX
		MOVQ src_base+24(FP), SI
		MOVQ src_len+32(FP), R10
		// Sanity check that len(dst) >= len(src).
		CMPQ BX, R10
		JLT  {{.ShortName}}End
		`

	fxSetup = ``
	flSetup = `
		// Prepare to set MXCSR bits 13 and 14, so that the CVTPS2PL below is
		// "Round To Zero".
		STMXCSR mxcsrOrig-8(SP)
		MOVL    mxcsrOrig-8(SP), AX
		ORL     $0x6000, AX
		MOVL    AX, mxcsrNew-4(SP)
		`

	fxLoadXMMRegs = `
		// fxAlmost65536 := XMM(0x0000ffff repeated four times) // Maximum of an uint16.
		MOVOU fxAlmost65536<>(SB), X5
		`
	flLoadXMMRegs = `
		// flSignMask    := XMM(0x7fffffff repeated four times) // All but the sign bit of a float32.
		// flOne         := XMM(0x3f800000 repeated four times) // 1 as a float32.
		// flAlmost65536 := XMM(0x477fffff repeated four times) // 255.99998 * 256 as a float32.
		MOVOU flSignMask<>(SB), X3
		MOVOU flOne<>(SB), X4
		MOVOU flAlmost65536<>(SB), X5
		`

	fxAdd = `PADDD`
	flAdd = `ADDPS`

	fxClampAndScale = `
		// y = abs(x)
		// y >>= 2 // Shift by 2*ϕ - 16.
		// y = min(y, fxAlmost65536)
		PABSD  X1, X2
		PSRLL  $2, X2
		PMINUD X5, X2
		`
	flClampAndScale = `
		// y = x & flSignMask
		// y = min(y, flOne)
		// y = mul(y, flAlmost65536)
		MOVOU X3, X2
		ANDPS X1, X2
		MINPS X4, X2
		MULPS X5, X2
		`

	fxConvertToInt32 = `
		// z = convertToInt32(y)
		// No-op.
		`
	flConvertToInt32 = `
		// z = convertToInt32(y)
		LDMXCSR  mxcsrNew-4(SP)
		CVTPS2PL X2, X2
		LDMXCSR  mxcsrOrig-8(SP)
		`

	opOverStore4 = `
		// Blend over the dst's prior value. SIMD for i in 0..3:
		//
		// dstA := uint32(dst[i]) * 0x101
		// maskA := z@i
		// outA := dstA*(0xffff-maskA)/0xffff + maskA
		// dst[i] = uint8(outA >> 8)
		//
		// First, set X0 to dstA*(0xfff-maskA).
		MOVL   (DI), X0
		PSHUFB X8, X0
		MOVOU  X9, X11
		PSUBL  X2, X11
		PMULLD X11, X0
		// We implement uint32 division by 0xffff as multiplication by a magic
		// constant (0x800080001) and then a shift by a magic constant (47).
		// See TestDivideByFFFF for a justification.
		//
		// That multiplication widens from uint32 to uint64, so we have to
		// duplicate and shift our four uint32s from one XMM register (X0) to
		// two XMM registers (X0 and X11).
		//
		// Move the second and fourth uint32s in X0 to be the first and third
		// uint32s in X11.
		MOVOU X0, X11
		PSRLQ $32, X11
		// Multiply by magic, shift by magic.
		PMULULQ X10, X0
		PMULULQ X10, X11
		PSRLQ   $47, X0
		PSRLQ   $47, X11
		// Merge the two registers back to one, X11, and add maskA.
		PSLLQ $32, X11
		XORPS X0, X11
		PADDD X11, X2
		// As per opSrcStore4, shuffle and copy the 4 second-lowest bytes.
		PSHUFB X6, X2
		MOVL   X2, (DI)
		`
	opSrcStore4 = `
		// z = shuffleTheSecondLowestBytesOfEach4ByteElement(z)
		// copy(dst[:4], low4BytesOf(z))
		PSHUFB X6, X2
		MOVL   X2, (DI)
		`
	maskStore4 = `
		// copy(dst[:4], z)
		MOVOU X2, (DI)
		`

	opOverStore1 = `
		// Blend over the dst's prior value.
		//
		// dstA := uint32(dst[0]) * 0x101
		// maskA := z
		// outA := dstA*(0xffff-maskA)/0xffff + maskA
		// dst[0] = uint8(outA >> 8)
		MOVBLZX (DI), R12
		IMULL   $0x101, R12
		MOVL    X2, R13
		MOVL    $0xffff, AX
		SUBL    R13, AX
		MULL    R12             // MULL's implicit arg is AX, and the result is stored in DX:AX.
		MOVL    $0x80008001, BX // Divide by 0xffff is to first multiply by a magic constant...
		MULL    BX              // MULL's implicit arg is AX, and the result is stored in DX:AX.
		SHRL    $15, DX         // ...and then shift by another magic constant (47 - 32 = 15).
		ADDL    DX, R13
		SHRL    $8, R13
		MOVB    R13, (DI)
		`
	opSrcStore1 = `
		// dst[0] = uint8(z>>8)
		MOVL X2, BX
		SHRL $8, BX
		MOVB BX, (DI)
		`
	maskStore1 = `
		// dst[0] = uint32(z)
		MOVL X2, (DI)
		`

	opOverXMM6 = `gather`
	opSrcXMM6  = `gather`
	maskXMM6   = `-`

	opOverXMM8 = `scatterAndMulBy0x101`
	opSrcXMM8  = `-`
	maskXMM8   = `-`

	opOverXMM9 = `fxAlmost65536`
	opSrcXMM9  = `-`
	maskXMM9   = `-`

	opOverXMM10 = `inverseFFFF`
	opSrcXMM10  = `-`
	maskXMM10   = `-`

	opOverLoadXMMRegs = `
		// gather               := XMM(see above)                      // PSHUFB shuffle mask.
		// scatterAndMulBy0x101 := XMM(see above)                      // PSHUFB shuffle mask.
		// fxAlmost65536        := XMM(0x0000ffff repeated four times) // 0xffff.
		// inverseFFFF          := XMM(0x80008001 repeated four times) // Magic constant for dividing by 0xffff.
		MOVOU gather<>(SB), X6
		MOVOU scatterAndMulBy0x101<>(SB), X8
		MOVOU fxAlmost65536<>(SB), X9
		MOVOU inverseFFFF<>(SB), X10
		`
	opSrcLoadXMMRegs = `
		// gather := XMM(see above) // PSHUFB shuffle mask.
		MOVOU gather<>(SB), X6
		`
	maskLoadXMMRegs = ``
)