internal/export/idna: update for post-Unicode 10 idna changes

Unicode 11 changed the test file format, and we've never updated
to the new format nor kept up with algorithmic changes.

Unicode 16 dropped the disallowed_STD3 variants from the
IdnaMappingTable.txt file and made more algorithmic changes.

Update to the new test format and implement the new algorithm.
Note that unicode16 is a constant, so disabled version-specific
code is dead code.

For golang/go#77266.

Change-Id: Ic9d0a6a7a5922c12383de12264c2a6d38b951047
Reviewed-on: https://go-review.googlesource.com/c/text/+/737402
Reviewed-by: Roland Shoemaker <roland@golang.org>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>

Author: rsc

internal/export/idna/common_test.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+//go:build !go1.26
+
 package idna
 
 import (

internal/export/idna/conformance_test.go

@@ -0,0 +1,51 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.26
+
+package idna
+
+import (
+	"testing"
+
+	"golang.org/x/text/internal/gen"
+	"golang.org/x/text/internal/testtext"
+	"golang.org/x/text/internal/ucd"
+)
+
+func TestConformance(t *testing.T) {
+	testtext.SkipIfNotLong(t)
+
+	r := gen.OpenUnicodeFile("idna", "", "IdnaTestV2.txt")
+	defer r.Close()
+
+	section := "main"
+	p := ucd.New(r)
+	transitional := New(Transitional(true), VerifyDNSLength(true), BidiRule(), MapForLookup())
+	nonTransitional := New(VerifyDNSLength(true), BidiRule(), MapForLookup())
+	for p.Next() {
+		var (
+			src          = def(unescape(p.String(0)), "")
+			toUnicode    = def(unescape(p.String(1)), src)
+			toUnicodeErr = p.String(2)
+			toASCIIN     = def(unescape(p.String(3)), toUnicode)
+			toASCIINErr  = def(p.String(4), toUnicodeErr)
+			toASCIIT     = def(unescape(p.String(5)), toASCIIN)
+			toASCIITErr  = def(p.String(6), toASCIINErr)
+		)
+		doTest(t, nonTransitional.ToUnicode, section+":ToUnicode", src, toUnicode, toUnicodeErr)
+		doTest(t, nonTransitional.ToASCII, section+":ToASCII:N", src, toASCIIN, toASCIINErr)
+		doTest(t, transitional.ToASCII, section+":ToASCII:T", src, toASCIIT, toASCIITErr)
+	}
+}
+
+func def(field, fallback string) string {
+	if field == "" {
+		return fallback
+	}
+	if field == `""` {
+		return ""
+	}
+	return field
+}

internal/export/idna/conformancev2_test.go

@@ -279,3 +279,7 @@ func (c *normCompacter) Print(w io.Writer) (retErr error) {
 	p("\n}\n\n")
 	return
 }
+
+func allowedSTD3(r rune) bool {
+	return r >= 0x80 || '0' <= r && r <= '9' || 'a' <= r && r <= 'z' || r == '-' || r == '.'
+}

internal/export/idna/gen.go

@@ -16,7 +16,12 @@ import (
 )
 
 func catFromEntry(p *ucd.Parser) (cat category) {
+	// Note: As of Unicode 16, IdnaMappingTable.txt no longer includes
+	// disallowed_STD3_valid and disallowed_STD3_mapped.
+	// It is up to us to bring them back with our definition of disallowedSTD3,
+	// which is exactly the runes from Unicode 15.
 	r := p.Rune(0)
+	idna2008status := p.String(3)
 	switch s := p.String(1); s {
 	case "valid":
 		cat = valid
@@ -35,9 +40,9 @@ func catFromEntry(p *ucd.Parser) (cat category) {
 	default:
 		log.Fatalf("%U: Unknown category %q", r, s)
 	}
-	if s := p.String(3); s != "" {
+	if s := idna2008status; s != "" {
 		if cat != valid {
-			log.Fatalf(`%U: %s defined for %q; want "valid"`, r, s, p.String(1))
+			log.Fatalf(`%U: %s defined for %q/%v; want "valid"`, r, s, p.String(1), cat)
 		}
 		switch s {
 		case "NV8":