build changes (#2356)

* default build includes everything
* allow compiling out PRECIS and Skeleton (still included by default)
* consistently use `postgresql` in identifiers

Author: slingamn

.github/workflows/build.yml

@@ -22,8 +22,10 @@ jobs:
           go-version: "1.26"
       - name: "install python3-pytest"
         run: "sudo apt install -y python3-pytest python3-websockets"
-      - name: "make build_full"
-        run: "make build_full"
+      - name: "make minimal"
+        run: "make minimal"
+      - name: "make build"
+        run: "make build"
       - name: "make install"
         run: "make install"
       - name: "make test"

Makefile

@@ -5,13 +5,13 @@ GIT_TAG := $(shell git tag --points-at HEAD 2> /dev/null | head -n 1)
 # this can be overridden by passing CGO_ENABLED=1 to make
 export CGO_ENABLED ?= 0
 
-capdef_file = ./irc/caps/defs.go
+# build tags for the maximalist build with everything included
+full_tags = i18n mysql postgresql sqlite
 
-# default build tags; override by passing, e.g. ERGO_BUILD_TAGS="mysql postgres"
-ERGO_BUILD_TAGS ?= mysql
+# build everything by default; override by passing, e.g. ERGO_BUILD_TAGS="mysql postgresql"
+ERGO_BUILD_TAGS ?= $(full_tags)
 
-# build tags for the maximalist build with everything included
-full_tags = "mysql postgres sqlite"
+capdef_file = ./irc/caps/defs.go
 
 .PHONY: all
 all: build
@@ -20,31 +20,28 @@ all: build
 build:
 	go build -v -tags "$(ERGO_BUILD_TAGS)" -ldflags "-X main.commit=$(GIT_COMMIT) -X main.version=$(GIT_TAG)"
 
-.PHONY: build_full
-build_full:
-	go build -v -tags $(full_tags) -ldflags "-X main.commit=$(GIT_COMMIT) -X main.version=$(GIT_TAG)"
-
 .PHONY: install
 install:
-	go install -v -tags $(ERGO_BUILD_TAGS) -ldflags "-X main.commit=$(GIT_COMMIT) -X main.version=$(GIT_TAG)"
-
-.PHONY: install_full
-install_full:
-	go install -v -tags $(full_tags) -ldflags "-X main.commit=$(GIT_COMMIT) -X main.version=$(GIT_TAG)"
+	go install -v -tags "$(ERGO_BUILD_TAGS)" -ldflags "-X main.commit=$(GIT_COMMIT) -X main.version=$(GIT_TAG)"
 
 .PHONY: release
 release:
 	goreleaser --skip=publish --clean
 
+.PHONY: minimal
+minimal:
+	go build -v -tags "" -ldflags "-X main.commit=$(GIT_COMMIT) -X main.version=$(GIT_TAG)"
+
 .PHONY: capdefs
 capdefs:
 	python3 ./gencapdefs.py > ${capdef_file}
 
 .PHONY: test
 test:
 	python3 ./gencapdefs.py | diff - ${capdef_file}
-	go test -tags $(full_tags) ./...
-	go vet -tags $(full_tags) ./...
+	go test -tags "$(full_tags)" ./...
+	go vet -tags "$(full_tags)" ./...
+	go vet -tags "" ./...
 	./.check-gofmt.sh
 
 .PHONY: smoke

README.md

@@ -81,18 +81,13 @@ You can also clone this repository and build from source. A quick start guide:
 1. Obtain an [up-to-date distribution of the Go language for your OS and architecture](https://golang.org/dl/). Check the output of `go version` to ensure it was installed correctly.
 1. Clone the repository.
 1. `git checkout stable`
-1. To build the default Ergo binary, `make`
-1. To instead build an Ergo binary that includes support for all datastores (including PostgreSQL and SQLite), `make build_full`
+1. `make`
 1. You should now have a binary named `ergo` in the working directory.
 
-For more information, including on build customization, see [docs/BUILD.md](https://github.com/ergochat/ergo/blob/master/docs/BUILD.md).
+Ergo vendors all its dependencies, so you will not need to fetch any dependencies remotely. For more information, including on build customization, see [docs/BUILD.md](https://github.com/ergochat/ergo/blob/master/docs/BUILD.md).
 
 For information on contributing to Ergo, see [DEVELOPING.md](https://github.com/ergochat/ergo/blob/master/DEVELOPING.md).
 
-#### Building
-
-You'll need an [up-to-date distribution of the Go language for your OS and architecture](https://golang.org/dl/). Once that's installed (check the output of `go version`), just check out your desired branch or tag and run `make`. This will produce an executable binary named `ergo` in the base directory of the project. (Ergo vendors all its dependencies, so you will not need to fetch any dependencies remotely.)
-
 ## Configuration
 
 The default config file [`default.yaml`](default.yaml) helps walk you through what each option means and changes.

docs/BUILD.md

@@ -31,4 +31,11 @@ The `master` branch is not recommended for production use since it may contain b
 
 By default, Ergo is built with cgo disabled, producing a fully statically linked binary. You can disable this with `export CGO_ENABLED=1` before running `make`.
 
-The default Ergo binary (built with `make` or `make build`) includes support for an in-memory history backend, plus a MySQL history backend. `make build_full` will additionally compile in support for PostgreSQL and SQLite history backends. You can also customize which backends are included, with, e.g. `export ERGO_BUILD_TAGS="mysql sqlite"`.
+The default Ergo binary (built with `make` or `make build`) includes support for all optional features. Each optional feature is controlled via a separate build tag; to override the build tags, pass the environment variable `ERGO_BUILD_TAGS` with a space-separated list of tags. (For example, for parity with v2.17.0 and earlier, you can run `ERGO_BUILD_TAGS="i18n mysql" make`. Passing the empty string disables all optional features.)
+
+The supported build tags are:
+
+* `i18n` enables support for non-ASCII casemappings (allowing Unicode in nicknames and channel names). (This was a default feature in Ergo v2.17.0 and earlier, but was not enabled by default at runtime. See the `server.casemapping` value of the config file.)
+* `mysql` enables support for MySQL as a persistent history backend. (This was a default feature in v2.17.0 and earlier.)
+* `postgresql` enables support for PostgreSQL as a persistent history backend.
+* `sqlite` enables support for SQLite as a persistent history backend.

irc/config.go

@@ -33,6 +33,7 @@ import (
 	"github.com/ergochat/ergo/irc/connection_limits"
 	"github.com/ergochat/ergo/irc/custime"
 	"github.com/ergochat/ergo/irc/email"
+	"github.com/ergochat/ergo/irc/i18n"
 	"github.com/ergochat/ergo/irc/isupport"
 	"github.com/ergochat/ergo/irc/jwt"
 	"github.com/ergochat/ergo/irc/languages"
@@ -41,7 +42,7 @@ import (
 	"github.com/ergochat/ergo/irc/mysql"
 	"github.com/ergochat/ergo/irc/oauth2"
 	"github.com/ergochat/ergo/irc/passwd"
-	"github.com/ergochat/ergo/irc/postgres"
+	"github.com/ergochat/ergo/irc/postgresql"
 	"github.com/ergochat/ergo/irc/sqlite"
 	"github.com/ergochat/ergo/irc/utils"
 	"github.com/ergochat/ergo/irc/webpush"
@@ -447,31 +448,6 @@ func (nr *NickEnforcementMethod) UnmarshalYAML(unmarshal func(interface{}) error
 	return err
 }
 
-func (cm *Casemapping) UnmarshalYAML(unmarshal func(interface{}) error) (err error) {
-	var orig string
-	if err = unmarshal(&orig); err != nil {
-		return err
-	}
-
-	var result Casemapping
-	switch strings.ToLower(orig) {
-	case "ascii":
-		result = CasemappingASCII
-	case "precis", "rfc7613", "rfc8265":
-		result = CasemappingPRECIS
-	case "permissive", "fun":
-		result = CasemappingPermissive
-	case "rfc1459":
-		result = CasemappingRFC1459
-	case "rfc1459-strict":
-		result = CasemappingRFC1459Strict
-	default:
-		return fmt.Errorf("invalid casemapping value: %s", orig)
-	}
-	*cm = result
-	return nil
-}
-
 // OperClassConfig defines a specific operator class.
 type OperClassConfig struct {
 	Title        string
@@ -615,7 +591,7 @@ type Config struct {
 		supportedCaps            *caps.Set
 		supportedCapsWithoutSTS  *caps.Set
 		capValues                caps.Values
-		Casemapping              Casemapping
+		Casemapping              i18n.Casemapping
 		EnforceUtf8              bool                `yaml:"enforce-utf8"`
 		OutputPath               string              `yaml:"output-path"`
 		IPCheckScript            IPCheckScriptConfig `yaml:"ip-check-script"`
@@ -664,7 +640,7 @@ type Config struct {
 		Path        string
 		AutoUpgrade bool
 		MySQL       mysql.Config
-		PostgreSQL  postgres.Config
+		PostgreSQL  postgresql.Config
 		SQLite      sqlite.Config
 	}
 
@@ -1255,11 +1231,11 @@ func LoadConfig(filename string) (config *Config, err error) {
 		}
 	}
 	if config.Datastore.PostgreSQL.Enabled {
-		if !postgres.Enabled {
+		if !postgresql.Enabled {
 			return nil, fmt.Errorf("PostgreSQL is enabled in the config, but this binary was not built with PostgreSQL support. Rebuild with `make build_full` to enable")
 		}
-		if config.Limits.NickLen > postgres.MaxTargetLength || config.Limits.ChannelLen > postgres.MaxTargetLength {
-			return nil, fmt.Errorf("to use PostgreSQL, nick and channel length limits must be %d or lower", postgres.MaxTargetLength)
+		if config.Limits.NickLen > postgresql.MaxTargetLength || config.Limits.ChannelLen > postgresql.MaxTargetLength {
+			return nil, fmt.Errorf("to use PostgreSQL, nick and channel length limits must be %d or lower", postgresql.MaxTargetLength)
 		}
 	}
 	if config.Datastore.SQLite.Enabled {
@@ -1377,6 +1353,12 @@ func LoadConfig(filename string) (config *Config, err error) {
 		config.Server.capValues[caps.Multiline] = multilineCapValue
 	}
 
+	if !i18n.Enabled {
+		if config.Server.Casemapping != i18n.CasemappingASCII {
+			return nil, fmt.Errorf("i18n support was compiled out; set casemapping to 'ascii' or recompile")
+		}
+	}
+
 	// handle legacy name 'bouncer' for 'multiclient' section:
 	if config.Accounts.Bouncer != nil {
 		config.Accounts.Multiclient = *config.Accounts.Bouncer
@@ -1711,7 +1693,7 @@ func LoadConfig(filename string) (config *Config, err error) {
 		// same machine:
 		config.Datastore.MySQL.MaxConns = runtime.NumCPU()
 	}
-	// do the same for postgres
+	// do the same for postgresql
 	config.Datastore.PostgreSQL.ExpireTime = time.Duration(config.History.Restrictions.ExpireTime)
 	config.Datastore.PostgreSQL.TrackAccountMessages = config.History.Retention.EnableAccountIndexing
 	if config.Datastore.PostgreSQL.MaxConns == 0 {
@@ -1835,9 +1817,9 @@ func (config *Config) generateISupport() (err error) {
 	switch config.Server.Casemapping {
 	default:
 		casemappingToken = "ascii" // this is published for ascii, precis, or permissive
-	case CasemappingRFC1459:
+	case i18n.CasemappingRFC1459:
 		casemappingToken = "rfc1459"
-	case CasemappingRFC1459Strict:
+	case i18n.CasemappingRFC1459Strict:
 		casemappingToken = "rfc1459-strict"
 	}
 	isupport.Add("CASEMAPPING", casemappingToken)
@@ -1876,7 +1858,7 @@ func (config *Config) generateISupport() (err error) {
 	isupport.Add("STATUSMSG", "~&@%+")
 	isupport.Add("TARGMAX", fmt.Sprintf("NAMES:1,LIST:1,KICK:,WHOIS:1,USERHOST:10,PRIVMSG:%s,TAGMSG:%s,NOTICE:%s,MONITOR:%d", maxTargetsString, maxTargetsString, maxTargetsString, config.Limits.MonitorEntries))
 	isupport.Add("TOPICLEN", strconv.Itoa(config.Limits.TopicLen))
-	if config.Server.Casemapping == CasemappingPRECIS {
+	if config.Server.Casemapping == i18n.CasemappingPRECIS {
 		isupport.Add("UTF8MAPPING", precisUTF8MappingToken)
 	}
 	if config.Server.EnforceUtf8 {
@@ -1953,7 +1935,7 @@ func (config *Config) historyChangedFrom(oldConfig *Config) bool {
 		config.History.Persistent != oldConfig.History.Persistent
 }
 
-func compileGuestRegexp(guestFormat string, casemapping Casemapping) (standard, folded *regexp.Regexp, err error) {
+func compileGuestRegexp(guestFormat string, casemapping i18n.Casemapping) (standard, folded *regexp.Regexp, err error) {
 	if strings.Count(guestFormat, "?") != 0 || strings.Count(guestFormat, "*") != 1 {
 		err = errors.New("guest format must contain 1 '*' and no '?'s")
 		return
@@ -1967,11 +1949,11 @@ func compileGuestRegexp(guestFormat string, casemapping Casemapping) (standard,
 	starIndex := strings.IndexByte(guestFormat, '*')
 	initial := guestFormat[:starIndex]
 	final := guestFormat[starIndex+1:]
-	initialFolded, err := casefoldWithSetting(initial, casemapping)
+	initialFolded, err := i18n.CasefoldWithSetting(initial, casemapping)
 	if err != nil {
 		return
 	}
-	finalFolded, err := casefoldWithSetting(final, casemapping)
+	finalFolded, err := i18n.CasefoldWithSetting(final, casemapping)
 	if err != nil {
 		return
 	}

irc/errors.go

@@ -82,9 +82,8 @@ var (
 
 // String Errors
 var (
-	errCouldNotStabilize = errors.New("Could not stabilize string while casefolding")
-	errStringIsEmpty     = errors.New("String is empty")
-	errInvalidCharacter  = errors.New("Invalid character")
+	errStringIsEmpty    = errors.New("String is empty")
+	errInvalidCharacter = errors.New("Invalid character")
 )
 
 type CertKeyError struct {

irc/i18n/common.go

@@ -0,0 +1,79 @@
+package i18n
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+)
+
+// Casemapping represents a set of algorithm for case normalization
+// and confusables prevention for IRC identifiers (nicknames and channel names)
+type Casemapping uint
+
+const (
+	// "precis" is the default / zero value:
+	// casefolding/validation: PRECIS + ircd restrictions (like no *)
+	// confusables detection: standard skeleton algorithm
+	CasemappingPRECIS Casemapping = iota
+	// "ascii" is the traditional ircd behavior:
+	// casefolding/validation: must be pure ASCII and follow ircd restrictions, ASCII lowercasing
+	// confusables detection: none
+	CasemappingASCII
+	// "permissive" is an insecure mode:
+	// casefolding/validation: arbitrary unicodes that follow ircd restrictions, unicode casefolding
+	// confusables detection: standard skeleton algorithm (which may be ineffective
+	// over the larger set of permitted identifiers)
+	CasemappingPermissive
+	// rfc1459 is a legacy mapping as defined here: https://modern.ircdocs.horse/#casemapping-parameter
+	CasemappingRFC1459
+	// rfc1459-strict is a legacy mapping as defined here: https://modern.ircdocs.horse/#casemapping-parameter
+	CasemappingRFC1459Strict
+)
+
+var (
+	errInvalidCharacter = errors.New("Invalid character")
+)
+
+func (cm *Casemapping) UnmarshalYAML(unmarshal func(interface{}) error) (err error) {
+	var orig string
+	if err = unmarshal(&orig); err != nil {
+		return err
+	}
+
+	var result Casemapping
+	switch strings.ToLower(orig) {
+	case "ascii":
+		result = CasemappingASCII
+	case "precis", "rfc7613", "rfc8265":
+		result = CasemappingPRECIS
+	case "permissive", "fun":
+		result = CasemappingPermissive
+	case "rfc1459":
+		result = CasemappingRFC1459
+	case "rfc1459-strict":
+		result = CasemappingRFC1459Strict
+	default:
+		return fmt.Errorf("invalid casemapping value: %s", orig)
+	}
+	*cm = result
+	return nil
+}
+
+func isPrintableASCII(str string) bool {
+	for i := 0; i < len(str); i++ {
+		// allow space here because it's technically printable;
+		// it will be disallowed later by CasefoldName/CasefoldChannel
+		chr := str[i]
+		if chr < ' ' || chr > '~' {
+			return false
+		}
+	}
+	return true
+}
+
+func foldASCII(str string) (result string, err error) {
+	if !isPrintableASCII(str) {
+		return "", errInvalidCharacter
+	}
+	return strings.ToLower(str), nil
+}