Skip to content

Commit e389f75

Browse files
AlizterAlizter
committed
feat(pkg): relocatable compiler support
Signed-off-by: Ali Caglayan <alizter@gmail.com>
1 parent 752a6d6 commit e389f75

File tree

4 files changed

+198
-23
lines changed

4 files changed

+198
-23
lines changed

‎src/dune_rules/pkg_rules.ml‎

Lines changed: 47 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -527,17 +527,6 @@ module Pkg = struct
527527
|> List.fold_left ~init:Dep.Set.empty ~f:(fun acc t -> dep t |> Dep.Set.add acc)
528528
;;
529529

530-
let install_roots t =
531-
let default_install_roots = Paths.install_roots t.paths in
532-
match Pkg_toolchain.is_compiler_and_toolchains_enabled t.info.name with
533-
| false -> default_install_roots
534-
| true ->
535-
(* Compiler packages store their libraries in a subdirectory named "ocaml". *)
536-
{ default_install_roots with
537-
lib_root = Path.relative default_install_roots.lib_root "ocaml"
538-
}
539-
;;
540-
541530
(* Given a list of packages, construct an env containing variables
542531
set by each package. Variables containing delimited lists of
543532
paths (e.g. PATH) which appear in multiple package's envs are
@@ -548,7 +537,7 @@ module Pkg = struct
548537
let build_env_of_deps ts =
549538
List.fold_left ts ~init:Env.Map.empty ~f:(fun env t ->
550539
let env =
551-
let roots = install_roots t in
540+
let roots = Paths.install_roots t.paths in
552541
let init = Value_list_env.add_path env Env_path.var roots.bin in
553542
let vars = Install.Roots.to_env_without_path roots ~relative:Path.relative in
554543
List.fold_left vars ~init ~f:(fun acc (var, path) ->
@@ -1356,12 +1345,42 @@ module DB = struct
13561345
{ id : Id.t
13571346
; pkg_digest_table : Pkg_table.t
13581347
; system_provided : Package.Name.Set.t
1348+
; is_relocatable_compiler_context : bool
13591349
}
13601350

13611351
let equal x y = Id.equal x.id y.id
13621352

1363-
let create ~pkg_digest_table ~system_provided =
1364-
{ id = Id.gen (); pkg_digest_table; system_provided }
1353+
let create =
1354+
(* In dra27's relocatable repository, a compiler is relocatable if it
1355+
depends on this meta-package. *)
1356+
let[@inline] is_relocatable_compiler_meta_package name =
1357+
Package.Name.equal name (Package.Name.of_string "relocatable-compiler")
1358+
in
1359+
(* OCaml 5.5.0+ is natively relocatable. *)
1360+
let[@inline] is_ocaml_5_5_or_above name version =
1361+
Dune_pkg.Dev_tool.is_compiler_package name
1362+
&& OpamPackage.Version.compare
1363+
(Dune_pkg.Package_version.to_opam_package_version version)
1364+
(OpamPackage.Version.of_string "5.5.0")
1365+
|> Ordering.of_int
1366+
|> function
1367+
| Gt | Eq -> true
1368+
| Lt -> false
1369+
in
1370+
fun ~pkg_digest_table ~system_provided ->
1371+
{ id = Id.gen ()
1372+
; pkg_digest_table
1373+
; system_provided
1374+
(* To know if a given package table has the relocatable compiler, we
1375+
need to find either the "relocatable-compiler" meta-package or
1376+
assert that the version of the OCaml compiler is at least 5.5.0. *)
1377+
; is_relocatable_compiler_context =
1378+
Pkg_digest.Map.existsi
1379+
pkg_digest_table
1380+
~f:(fun _ { Pkg_table.pkg = { info = { name; version; _ }; _ }; _ } ->
1381+
is_relocatable_compiler_meta_package name
1382+
|| is_ocaml_5_5_or_above name version)
1383+
}
13651384
;;
13661385

13671386
let pkg_digest_of_name lock_dir platform pkg_name ~system_provided =
@@ -1590,9 +1609,11 @@ end = struct
15901609
let build_command = Option.map build_command ~f:relocate_build in
15911610
let paths =
15921611
let paths = Paths.map_path write_paths ~f:Path.build in
1593-
match Pkg_toolchain.is_compiler_and_toolchains_enabled info.name with
1594-
| false -> paths
1595-
| true ->
1612+
if
1613+
db.is_relocatable_compiler_context
1614+
|| not (Pkg_toolchain.is_compiler_package_with_toolchains_enabled info.name)
1615+
then paths
1616+
else (
15961617
(* Modify the environment as well as build and install commands for
15971618
the compiler package. The specific changes are:
15981619
- setting the prefix in the build environment to inside the user's
@@ -1603,16 +1624,21 @@ end = struct
16031624
toolchain directory
16041625
- if a matching version of the compiler is
16051626
already installed in the user's toolchain directory then the
1606-
build and install commands are replaced with no-ops *)
1627+
build and install commands are replaced with no-ops
1628+
- compiler packages store their libraries in a subdirectory
1629+
named "ocaml" *)
16071630
let prefix = Pkg_toolchain.installation_prefix pkg in
16081631
let install_roots =
1609-
Pkg_toolchain.install_roots ~prefix
1610-
|> Install.Roots.map ~f:Path.outside_build_dir
1632+
let roots =
1633+
Pkg_toolchain.install_roots ~prefix
1634+
|> Install.Roots.map ~f:Path.outside_build_dir
1635+
in
1636+
{ roots with lib_root = Path.relative roots.lib_root "ocaml" }
16111637
in
16121638
{ paths with
16131639
prefix = Path.outside_build_dir prefix
16141640
; install_roots = Lazy.from_val install_roots
1615-
}
1641+
})
16161642
in
16171643
let t =
16181644
{ Pkg.id

‎src/dune_rules/pkg_toolchain.ml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ let installation_prefix pkg =
4747
Path.Outside_build_dir.relative pkg_dir "target"
4848
;;
4949

50-
let is_compiler_and_toolchains_enabled name =
50+
let is_compiler_package_with_toolchains_enabled name =
5151
match Config.get Compile_time.toolchains with
5252
| `Enabled -> Dune_pkg.Dev_tool.is_compiler_package name
5353
| `Disabled -> false

‎src/dune_rules/pkg_toolchain.mli‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ val base_dir : unit -> Path.Outside_build_dir.t
1818
manage their compiler installation with opam or a system package
1919
manager, as compilers packages that would be installed by dune will
2020
not work correctly. *)
21-
val is_compiler_and_toolchains_enabled : Package.Name.t -> bool
21+
val is_compiler_package_with_toolchains_enabled : Package.Name.t -> bool
2222

2323
(** Returns the path to the directory containing the given package within the
2424
toolchain directory. This will be something like

‎test/blackbox-tests/test-cases/pkg/relocatable-compiler.t‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
Test that relocatable compilers bypass toolchain cache. This includes:
2+
- OCaml < 5.5 with the relocatable-compiler package
3+
- OCaml >= 5.5 which is natively relocatable
4+
5+
This mirrors the package layout from
6+
https://github.com/dra27/opam-repository/tree/relocatable
7+
8+
The relocatable repo is separate from the main opam repo and should take priority.
9+
10+
$ mkrepo
11+
$ mkdir -p relocatable-repo/packages
12+
13+
Create the relocatable compiler packages in separate repo:
14+
15+
$ mkdir -p relocatable-repo/packages/compiler-cloning/compiler-cloning.0.0.1
16+
$ cat > relocatable-repo/packages/compiler-cloning/compiler-cloning.0.0.1/opam << 'EOF'
17+
> opam-version: "2.0"
18+
> EOF
19+
20+
$ mkdir -p relocatable-repo/packages/relocatable-compiler/relocatable-compiler.5.4.0
21+
$ cat > relocatable-repo/packages/relocatable-compiler/relocatable-compiler.5.4.0/opam << 'EOF'
22+
> opam-version: "2.0"
23+
> depends: [ "compiler-cloning" ]
24+
> build: [ "sh" "-c" "echo %{_:build-id}% > build-id.txt" ]
25+
> install: [ "cp" "build-id.txt" "%{lib}%/relocatable-compiler/" ]
26+
> EOF
27+
28+
$ mkdir -p relocatable-repo/packages/ocaml-base-compiler/ocaml-base-compiler.5.4.0
29+
$ cat > relocatable-repo/packages/ocaml-base-compiler/ocaml-base-compiler.5.4.0/opam << 'EOF'
30+
> opam-version: "2.0"
31+
> depends: [ "relocatable-compiler" {= "5.4.0"} ]
32+
> build: [
33+
> [ "sh" "-c" "echo 'ocamlc binary' > ocamlc.opt" ]
34+
> [ "ln" "-s" "ocamlc.opt" "ocamlc" ]
35+
> ]
36+
> install: [
37+
> [ "mkdir" "-p" "%{bin}%" ]
38+
> [ "cp" "ocamlc.opt" "%{bin}%/" ]
39+
> [ "cp" "-P" "ocamlc" "%{bin}%/" ]
40+
> ]
41+
> EOF
42+
43+
$ mkdir -p relocatable-repo/packages/ocaml/ocaml.5.4.0
44+
$ cat > relocatable-repo/packages/ocaml/ocaml.5.4.0/opam << 'EOF'
45+
> opam-version: "2.0"
46+
> depends: [ "ocaml-base-compiler" {= "5.4.0"} ]
47+
> EOF
48+
49+
Also create a standard ocaml-base-compiler 5.4 in mock repo. This tests that
50+
the relocatable repo takes priority:
51+
52+
$ mkpkg ocaml-base-compiler 5.4.0
53+
54+
$ mkpkg ocaml 5.4.0 << 'EOF'
55+
> depends: [ "ocaml-base-compiler" {= "5.4.0"} ]
56+
> EOF
57+
58+
$ cat > dune-project << 'EOF'
59+
> (lang dune 3.22)
60+
> (package
61+
> (name test)
62+
> (allow_empty)
63+
> (depends ocaml))
64+
> EOF
65+
66+
$ cat > dune-workspace << EOF
67+
> (lang dune 3.20)
68+
> (lock_dir
69+
> (repositories relocatable-repo mock)) ; Order here is important here
70+
> (repository
71+
> (name relocatable-repo)
72+
> (url "file://$PWD/relocatable-repo"))
73+
> (repository
74+
> (name mock)
75+
> (url "file://$PWD/mock-opam-repository"))
76+
> EOF
77+
78+
We enable caching to verify that packages are built normally and get hardlinked
79+
into the cache. A hardlink count > 1 proves the file was cached.
80+
81+
$ export DUNE_CACHE=enabled
82+
$ export DUNE_CACHE_ROOT=$PWD/dune-cache
83+
84+
Solving for OCaml 5.4 should pick up the relocatable compiler.
85+
86+
$ dune_pkg_lock_normalized
87+
Solution for dune.lock:
88+
- compiler-cloning.0.0.1
89+
- ocaml.5.4.0
90+
- ocaml-base-compiler.5.4.0
91+
- relocatable-compiler.5.4.0
92+
93+
$ build_pkg ocaml-base-compiler
94+
95+
Toolchain cache should be empty:
96+
97+
$ test -d dune-cache/toolchains
98+
[1]
99+
100+
Check the installed files from the compiler are hardlinked (cached as regular
101+
package):
102+
103+
$ dune_cmd stat hardlinks $(get_build_pkg_dir ocaml-base-compiler)/target/bin/ocamlc.opt
104+
3
105+
106+
For OCaml >= 5.5, relocatable is built-in, so no special packages needed. These
107+
go in the main opam repo.
108+
109+
$ mkpkg ocaml-base-compiler 5.5.0 << 'EOF'
110+
> flags: compiler
111+
> build: [ "sh" "-c" "echo 'native relocatable' > marker.txt" ]
112+
> install: [
113+
> [ "mkdir" "-p" "%{lib}%/ocaml" ]
114+
> [ "cp" "marker.txt" "%{lib}%/ocaml/" ]
115+
> ]
116+
> EOF
117+
118+
$ mkpkg ocaml 5.5.0 << 'EOF'
119+
> depends: [ "ocaml-base-compiler" {= "5.5.0"} ]
120+
> EOF
121+
122+
$ cat > dune-project << 'EOF'
123+
> (lang dune 3.16)
124+
> (package
125+
> (name test)
126+
> (allow_empty)
127+
> (depends (ocaml (= 5.5.0))))
128+
> EOF
129+
130+
The relocatable repo will not have OCaml >= 5.5 so we do not need to bother
131+
updating the repositories in the workspace.
132+
133+
$ dune_pkg_lock_normalized
134+
Solution for dune.lock:
135+
- ocaml.5.5.0
136+
- ocaml-base-compiler.5.5.0
137+
138+
$ build_pkg ocaml-base-compiler
139+
140+
Toolchain cache should be empty:
141+
142+
$ test -d dune-cache/toolchains
143+
[1]
144+
145+
Check the installed files from the compiler are hardlinked (cached as regular
146+
package):
147+
148+
$ dune_cmd stat hardlinks $(get_build_pkg_dir ocaml-base-compiler)/target/lib/ocaml/marker.txt
149+
2

0 commit comments

Comments
 (0)