2025-04-22 12:36.31: New job: test ocurrent/ocaml-docs-ci https://github.com/ocurrent/ocaml-docs-ci.git#refs/heads/next-gen2 (cb095a367a8964bd594109f331419be2af01f996) (linux-x86_64:(lint-fmt))
Base: ocaml/opam:debian-12-ocaml-4.08@sha256:23c85daf4eb9a91d154cf2f9fed47f4f9b9fa788adc9c1fc7f0802e3dc06d7c4
ocamlformat version: version 0.26.0 (from opam)
To reproduce locally:
git clone --recursive "https://github.com/ocurrent/ocaml-docs-ci.git" -b "next-gen2" && cd "ocaml-docs-ci" && git reset --hard cb095a36
cat > Dockerfile <<'END-OF-DOCKERFILE'
FROM ocaml/opam:debian-12-ocaml-4.08@sha256:23c85daf4eb9a91d154cf2f9fed47f4f9b9fa788adc9c1fc7f0802e3dc06d7c4
USER 1000:1000
RUN cd ~/opam-repository && (git cat-file -e 1b767795a4e028a4d45f916f1598c24ee7a7b6cb || git fetch origin master) && git reset -q --hard 1b767795a4e028a4d45f916f1598c24ee7a7b6cb && git log --no-decorate -n1 --oneline && opam update -u
RUN opam depext -i dune
WORKDIR /src
RUN opam depext -i ocamlformat=0.26.0
COPY --chown=1000:1000 . /src/
RUN opam exec -- dune build @fmt --ignore-promoted-rules || (echo "dune build @fmt failed"; exit 2)
END-OF-DOCKERFILE
docker build .
END-REPRO-BLOCK
2025-04-22 12:36.31: Using cache hint "ocurrent/ocaml-docs-ci-ocaml/opam:debian-12-ocaml-4.08@sha256:23c85daf4eb9a91d154cf2f9fed47f4f9b9fa788adc9c1fc7f0802e3dc06d7c4-debian-12-4.08_opam-2.3-ocamlformat-1b767795a4e028a4d45f916f1598c24ee7a7b6cb"
2025-04-22 12:36.31: Using OBuilder spec:
((from ocaml/opam:debian-12-ocaml-4.08@sha256:23c85daf4eb9a91d154cf2f9fed47f4f9b9fa788adc9c1fc7f0802e3dc06d7c4)
(user (uid 1000) (gid 1000))
(run (cache (opam-archives (target /home/opam/.opam/download-cache)))
(network host)
(shell "cd ~/opam-repository && (git cat-file -e 1b767795a4e028a4d45f916f1598c24ee7a7b6cb || git fetch origin master) && git reset -q --hard 1b767795a4e028a4d45f916f1598c24ee7a7b6cb && git log --no-decorate -n1 --oneline && opam update -u"))
(run (cache (opam-archives (target /home/opam/.opam/download-cache)))
(network host)
(shell "opam depext -i dune"))
(workdir /src)
(run (cache (opam-archives (target /home/opam/.opam/download-cache)))
(network host)
(shell "opam depext -i ocamlformat=0.26.0"))
(copy (src .) (dst /src/))
(run (shell "opam exec -- dune build @fmt --ignore-promoted-rules || (echo \"dune build @fmt failed\"; exit 2)"))
)
2025-04-22 12:36.31: Waiting for resource in pool OCluster
2025-04-22 12:36.31: Waiting for worker…
2025-04-22 12:36.32: Got resource from pool OCluster
Building on clete.caelum.ci.dev
All commits already cached
HEAD is now at cb095a3 Remove blocked items from monitor page
(from ocaml/opam:debian-12-ocaml-4.08@sha256:23c85daf4eb9a91d154cf2f9fed47f4f9b9fa788adc9c1fc7f0802e3dc06d7c4)
2025-04-22 12:36.33 ---> using "44ab722c6d76cc4aa083f99994b6eb04bd97004720a37e6316ca14ec9a9ee4aa" from cache
/: (user (uid 1000) (gid 1000))
/: (run (cache (opam-archives (target /home/opam/.opam/download-cache)))
(network host)
(shell "cd ~/opam-repository && (git cat-file -e 1b767795a4e028a4d45f916f1598c24ee7a7b6cb || git fetch origin master) && git reset -q --hard 1b767795a4e028a4d45f916f1598c24ee7a7b6cb && git log --no-decorate -n1 --oneline && opam update -u"))
1b767795a4 Merge pull request #27704 from maiste/release-dune-3.18.0
<><> Updating package repositories ><><><><><><><><><><><><><><><><><><><><><><>
[default] Initialised
default (at git+file:///home/opam/opam-repository):
[INFO] opam 2.1 and 2.2 include many performance and security improvements over 2.0; please consider upgrading (https://opam.ocaml.org/doc/Install.html)
Everything as up-to-date as possible (run with --verbose to show unavailable upgrades).
However, you may "opam upgrade" these packages explicitly, which will ask permission to downgrade or uninstall the conflicting packages.
Nothing to do.
# Run eval $(opam env) to update the current shell environment
2025-04-22 12:37.45 ---> saved as "5af033ca5998041d246dc8498882e9dc146ac54b140907f2e1916f6f582b7193"
/: (run (cache (opam-archives (target /home/opam/.opam/download-cache)))
(network host)
(shell "opam depext -i dune"))
# Detecting depexts using vars: arch=x86_64, os=linux, os-distribution=debian, os-family=debian
# No extra OS packages requirements found.
# All required OS packages found.
# Now letting opam install the packages
The following actions will be performed:
- install dune 3.18.0
<><> Gathering sources ><><><><><><><><><><><><><><><><><><><><><><><><><><><><>
[dune.3.18.0] found in cache
<><> Processing actions <><><><><><><><><><><><><><><><><><><><><><><><><><><><>
-> installed dune.3.18.0
Done.
# Run eval $(opam env) to update the current shell environment
2025-04-22 12:38.50 ---> saved as "f3ca3dfc78385ac1381397f690e815b0455920d345180c72e1fc56947dfe3be3"
/: (workdir /src)
/src: (run (cache (opam-archives (target /home/opam/.opam/download-cache)))
(network host)
(shell "opam depext -i ocamlformat=0.26.0"))
# Detecting depexts using vars: arch=x86_64, os=linux, os-distribution=debian, os-family=debian
# No extra OS packages requirements found.
# All required OS packages found.
# Now letting opam install the packages
The following actions will be performed:
- install sexplib0 v0.14.0 [required by base]
- install cmdliner 1.3.0 [required by ocamlformat]
- install dune-build-info 3.18.0 [required by ocamlformat-lib]
- install menhirLib 20240715 [required by ocamlformat-lib]
- install menhirCST 20240715 [required by menhir]
- install ocamlbuild 0.16.1 [required by fpath, astring, uuseg]
- install menhirSdk 20240715 [required by ocamlformat-lib]
- install either 1.0.0 [required by ocamlformat-lib]
- install result 1.5 [required by ocamlformat-lib]
- install ocaml-version 4.0.0 [required by ocamlformat-lib]
- install camlp-streams 5.0.1 [required by ocamlformat-lib]
- install csexp 1.5.2 [required by ocamlformat-lib]
- install seq base [required by re]
- install ocamlfind 1.9.8 [required by ocp-indent, astring, fpath, uuseg]
- install fix 20230505 [required by ocamlformat-lib]
- install menhir 20240715 [required by ocamlformat-lib]
- install dune-configurator 3.18.0 [required by base]
- install re 1.11.0 [required by ocamlformat]
- install topkg 1.0.8 [required by fpath, astring, uuseg]
- install base-bytes base [required by ocp-indent]
- install base v0.14.3 [required by ocamlformat-lib]
- install uutf 1.0.4 [required by ocamlformat-lib]
- install astring 0.8.5 [required by ocamlformat-lib]
- install ocp-indent 1.8.1 [required by ocamlformat-lib]
- install stdio v0.14.0 [required by ocamlformat-lib]
- install uucp 15.0.0 [required by uuseg]
- install fpath 0.7.3 [required by ocamlformat-lib]
- install uuseg 15.0.0 [required by ocamlformat-lib]
- install ocamlformat-lib 0.26.0 [required by ocamlformat]
- install ocamlformat 0.26.0
===== 30 to install =====
<><> Gathering sources ><><><><><><><><><><><><><><><><><><><><><><><><><><><><>
[astring.0.8.5] found in cache
[base.v0.14.3] found in cache
[camlp-streams.5.0.1] found in cache
[cmdliner.1.3.0] found in cache
[csexp.1.5.2] found in cache
[dune-build-info.3.18.0] found in cache
[dune-configurator.3.18.0] found in cache
[either.1.0.0] found in cache
[fix.20230505] found in cache
[fpath.0.7.3] found in cache
[menhir.20240715] found in cache
[menhirCST.20240715] found in cache
[menhirLib.20240715] found in cache
[menhirSdk.20240715] found in cache
[ocaml-version.4.0.0] found in cache
[ocamlbuild.0.16.1] found in cache
[ocamlfind.1.9.8] found in cache
[ocamlformat.0.26.0] found in cache
[ocamlformat-lib.0.26.0] found in cache
[ocp-indent.1.8.1] found in cache
[re.1.11.0] found in cache
[result.1.5] found in cache
[sexplib0.v0.14.0] found in cache
[stdio.v0.14.0] found in cache
[topkg.1.0.8] found in cache
[uucp.15.0.0] found in cache
[uuseg.15.0.0] found in cache
[uutf.1.0.4] found in cache
<><> Processing actions <><><><><><><><><><><><><><><><><><><><><><><><><><><><>
-> installed seq.base
-> installed camlp-streams.5.0.1
-> installed csexp.1.5.2
-> installed either.1.0.0
-> installed fix.20230505
-> installed cmdliner.1.3.0
-> installed menhirCST.20240715
-> installed menhirLib.20240715
-> installed menhirSdk.20240715
-> installed ocaml-version.4.0.0
-> installed re.1.11.0
-> installed result.1.5
-> installed sexplib0.v0.14.0
-> installed dune-build-info.3.18.0
-> installed dune-configurator.3.18.0
-> installed ocamlfind.1.9.8
-> installed base-bytes.base
-> installed ocp-indent.1.8.1
-> installed ocamlbuild.0.16.1
-> installed base.v0.14.3
-> installed stdio.v0.14.0
-> installed topkg.1.0.8
-> installed uutf.1.0.4
-> installed astring.0.8.5
-> installed fpath.0.7.3
-> installed menhir.20240715
-> installed uucp.15.0.0
-> installed uuseg.15.0.0
-> installed ocamlformat-lib.0.26.0
-> installed ocamlformat.0.26.0
Done.
<><> ocp-indent.1.8.1 installed successfully ><><><><><><><><><><><><><><><><><>
=> This package requires additional configuration for use in editors. Install package 'user-setup', or manually:
* for Emacs, add these lines to ~/.emacs:
(add-to-list 'load-path "/home/opam/.opam/4.08/share/emacs/site-lisp")
(require 'ocp-indent)
* for Vim, add this line to ~/.vimrc:
set rtp^="/home/opam/.opam/4.08/share/ocp-indent/vim"
# Run eval $(opam env) to update the current shell environment
2025-04-22 12:40.09 ---> saved as "9d4e05d7215de4c09734aa004b02c994d19dec2363b796d29ddf14afeabf8120"
/src: (copy (src .) (dst /src/))
2025-04-22 12:40.09 ---> saved as "088799107dc9135160b6966064e80be37c3da6a0350bf603b7afaba8b1a84690"
/src: (run (shell "opam exec -- dune build @fmt --ignore-promoted-rules || (echo \"dune build @fmt failed\"; exit 2)"))
ocamlformat: ignoring "src/solver/solver.mli" (misplaced documentation comments - warning 50)
File "src/solver/solver.mli", line 6, characters 0-69:
6 | (** [test hash] runs a test with commit [hash] in opam-repository. *)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Warning 50 [unexpected-docstring]: ambiguous documentation comment
Hint: (Warning 50) This file contains a documentation comment (** ... *) that the OCaml compiler does not know how to attach to the AST. OCamlformat does not support these cases. You can find more information at: https://github.com/ocaml-ppx/ocamlformat#overview. If you'd like to disable this check and let ocamlformat make a choice (though it might not be consistent with the ocaml compilers and odoc), you can set the --no-comment-check option.
-> required by _build/default/src/solver/.formatted/solver.mli
-> required by alias src/solver/.formatted/fmt
-> required by alias src/solver/fmt
File "src/api/solver/worker.ml", line 1, characters 0-0:
diff --git a/_build/default/src/api/solver/worker.ml b/_build/default/src/api/solver/.formatted/worker.ml
index 4782905..9b6163a 100644
--- a/_build/default/src/api/solver/worker.ml
+++ b/_build/default/src/api/solver/.formatted/worker.ml
@@ -12,9 +12,11 @@ module Vars = struct
[@@deriving yojson]
end
-type solve_result =
- { compile_universes : (string * string * string list) list;
- link_universes: (string * string * string list) list; } [@@deriving yojson, ord]
+type solve_result = {
+ compile_universes : (string * string * string list) list;
+ link_universes : (string * string * string list) list;
+}
+[@@deriving yojson, ord]
(** A set of packages for a single build. *)
module Selection = struct
@@ -40,7 +42,15 @@ module Solve_request = struct
[@@deriving yojson]
let pp_constraint ppf (pkg, rel, vsn) =
- Format.fprintf ppf "%s%s%s" pkg (match rel with | `Eq -> "=" | `Geq -> ">=" | `Gt -> ">" | `Leq -> "<=" | `Lt -> "<" | `Neq -> "<>") vsn
+ Format.fprintf ppf "%s%s%s" pkg
+ (match rel with
+ | `Eq -> "="
+ | `Geq -> ">="
+ | `Gt -> ">"
+ | `Leq -> "<="
+ | `Lt -> "<"
+ | `Neq -> "<>")
+ vsn
end
(** The response from the solver. *)
File "src/solver/main.ml", line 1, characters 0-0:
diff --git a/_build/default/src/solver/main.ml b/_build/default/src/solver/.formatted/main.ml
index 52e95bb..05bda32 100644
--- a/_build/default/src/solver/main.ml
+++ b/_build/default/src/solver/.formatted/main.ml
@@ -76,8 +76,9 @@ let () =
export service ~on:Lwt_unix.stdin)
| [| _prog; "--worker"; hash |] ->
Solver.main (Git_unix.Store.Hash.of_hex hash)
- | [| _prog; "test"; |] ->
- Solver.test (Git_unix.Store.Hash.of_hex "ac01ad6037c0bdcca9f67fe49cd54475b585f9b2")
+ | [| _prog; "test" |] ->
+ Solver.test
+ (Git_unix.Store.Hash.of_hex "ac01ad6037c0bdcca9f67fe49cd54475b585f9b2")
| args ->
Fmt.failwith "Usage: ocaml-ci-solver (got %a)"
Fmt.(array (quote string))
File "src/lib/epoch.mli", line 1, characters 0-0:
diff --git a/_build/default/src/lib/epoch.mli b/_build/default/src/lib/.formatted/epoch.mli
index e31dd99..fa7fab7 100644
--- a/_build/default/src/lib/epoch.mli
+++ b/_build/default/src/lib/.formatted/epoch.mli
@@ -1,6 +1,5 @@
type t
val v : Config.t -> t
-
val digest : t -> string
val pp : t Fmt.t
File "src/lib/live.ml", line 1, characters 0-0:
diff --git a/_build/default/src/lib/live.ml b/_build/default/src/lib/.formatted/live.ml
index 9ae3877..1901f07 100644
--- a/_build/default/src/lib/live.ml
+++ b/_build/default/src/lib/.formatted/live.ml
@@ -4,9 +4,7 @@ let set_current ~ssh name generation =
let open Current.Syntax in
Current.component "Set current folder"
|> let> generation in
- let new_generation_folder =
- Storage.Base.generation_folder generation
- in
+ let new_generation_folder = Storage.Base.generation_folder generation in
let storage_folder =
Fpath.(of_string (Ssh.storage_folder ssh) |> Result.get_ok)
in
@@ -18,9 +16,7 @@ let set_live ~ssh name generation =
let open Current.Syntax in
Current.component "Set live folder"
|> let> generation in
- let new_generation_folder =
- Storage.Base.generation_folder generation
- in
+ let new_generation_folder = Storage.Base.generation_folder generation in
let storage_folder =
Fpath.(of_string (Ssh.storage_folder ssh) |> Result.get_ok)
in
@@ -30,6 +26,4 @@ let set_live ~ssh name generation =
let set_to ~ssh name generation =
Current.all
- [
- set_current ~ssh name generation; set_live ~ssh name generation;
- ]
+ [ set_current ~ssh name generation; set_live ~ssh name generation ]
File "src/lib/compile.mli", line 1, characters 0-0:
diff --git a/_build/default/src/lib/compile.mli b/_build/default/src/lib/.formatted/compile.mli
index a538cdf..4dc2238 100644
--- a/_build/default/src/lib/compile.mli
+++ b/_build/default/src/lib/.formatted/compile.mli
@@ -5,13 +5,19 @@
the compilation, link and html generation steps, outputting the results in
the compile/ and html/ folders. *)
-type hashes = { compile_hash : string option; linked_hash : string option; html_hash : string option }
+type hashes = {
+ compile_hash : string option;
+ linked_hash : string option;
+ html_hash : string option;
+}
type t
(** A compiled package *)
-type jobty = | CompileAndLink | CompileOnly | LinkOnly
-(** The type of compilation job *)
+type jobty =
+ | CompileAndLink
+ | CompileOnly
+ | LinkOnly (** The type of compilation job *)
val hashes : t -> hashes
(** Hash of the compiled artifacts *)
File "src/lib/live.mli", line 1, characters 0-0:
diff --git a/_build/default/src/lib/live.mli b/_build/default/src/lib/.formatted/live.mli
index 30fd180..d4e67fc 100644
--- a/_build/default/src/lib/live.mli
+++ b/_build/default/src/lib/.formatted/live.mli
@@ -1,5 +1 @@
-val set_to :
- ssh:Config.Ssh.t ->
- string ->
- Epoch.t Current.t ->
- unit Current.t
+val set_to : ssh:Config.Ssh.t -> string -> Epoch.t Current.t -> unit Current.t
File "src/lib/record.ml", line 1, characters 0-0:
diff --git a/_build/default/src/lib/record.ml b/_build/default/src/lib/.formatted/record.ml
index 7b71be8..c72f873 100644
--- a/_build/default/src/lib/record.ml
+++ b/_build/default/src/lib/.formatted/record.ml
@@ -1,4 +1,3 @@
-
let v config =
let open Lwt.Syntax in
let generation = Epoch.v config in
@@ -7,9 +6,7 @@ let v config =
let epoch_html = Epoch.digest generation in
let result =
- Index.record_new_pipeline
- ~odoc_commit ~epoch_html
- ~epoch_linked
+ Index.record_new_pipeline ~odoc_commit ~epoch_html ~epoch_linked
in
match result with
| Ok pipeline_id -> Lwt.return_ok (pipeline_id |> Int64.to_int)
File "src/lib/prep.mli", line 1, characters 0-0:
diff --git a/_build/default/src/lib/prep.mli b/_build/default/src/lib/.formatted/prep.mli
index ce07fe0..5e25d50 100644
--- a/_build/default/src/lib/prep.mli
+++ b/_build/default/src/lib/.formatted/prep.mli
@@ -1,8 +1,7 @@
type t
(** The type for a prepped package (build objects in a universe/package folder) *)
-module OpamFiles :sig
-
+module OpamFiles : sig
type t = No_context
module Key : sig
@@ -10,15 +9,17 @@ module OpamFiles :sig
end
module Value : sig
- type t = ((bool * string), [`Msg of string]) result OpamPackage.Map.t
+ type t = (bool * string, [ `Msg of string ]) result OpamPackage.Map.t
[@@deriving yojson]
end
end
module OpamFilesCache : sig
- val get : ?schedule:Current_cache.Schedule.t ->
+ val get :
+ ?schedule:Current_cache.Schedule.t ->
OpamFiles.t ->
- OpamFiles.Key.t -> OpamFiles.Value.t Current.Primitive.t
+ OpamFiles.Key.t ->
+ OpamFiles.Value.t Current.Primitive.t
end
val hash : t -> string
ocamlformat: ignoring "src/lib/misc.ml" (misplaced documentation comments - warning 50)
File "src/lib/misc.ml", line 64, characters 0-31:
64 | (** Select base image to use *)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Warning 50 [unexpected-docstring]: ambiguous documentation comment
Hint: (Warning 50) This file contains a documentation comment (** ... *) that the OCaml compiler does not know how to attach to the AST. OCamlformat does not support these cases. You can find more information at: https://github.com/ocaml-ppx/ocamlformat#overview. If you'd like to disable this check and let ocamlformat make a choice (though it might not be consistent with the ocaml compilers and odoc), you can set the --no-comment-check option.
-> required by _build/default/src/lib/.formatted/misc.ml
-> required by alias src/lib/.formatted/fmt
-> required by alias src/lib/fmt
File "src/lib/storage.mli", line 1, characters 0-0:
diff --git a/_build/default/src/lib/storage.mli b/_build/default/src/lib/.formatted/storage.mli
index 5230ae1..b86d55d 100644
--- a/_build/default/src/lib/storage.mli
+++ b/_build/default/src/lib/.formatted/storage.mli
@@ -8,7 +8,12 @@ type repository =
val folder : repository -> Package.t -> Fpath.t
module Base : sig
- type repository = HtmlRaw of Epoch.t | Linked of Epoch.t | Compile of Epoch.t | Prep | Prep0
+ type repository =
+ | HtmlRaw of Epoch.t
+ | Linked of Epoch.t
+ | Compile of Epoch.t
+ | Prep
+ | Prep0
val folder : repository -> Fpath.t
val generation_folder : Epoch.t -> Fpath.t
File "src/lib/epoch.ml", line 1, characters 0-0:
diff --git a/_build/default/src/lib/epoch.ml b/_build/default/src/lib/.formatted/epoch.ml
index d10e48a..1b22f57 100644
--- a/_build/default/src/lib/epoch.ml
+++ b/_build/default/src/lib/.formatted/epoch.ml
@@ -1,15 +1,15 @@
-type t = { config : Config.t; }
+type t = { config : Config.t }
let version = "v2"
-let v config = { config; }
+let v config = { config }
let digest t =
let key =
- Fmt.str "%s:%s:%s" version (Config.odoc t.config) (Config.sherlodoc t.config)
+ Fmt.str "%s:%s:%s" version (Config.odoc t.config)
+ (Config.sherlodoc t.config)
in
key |> Digest.string |> Digest.to_hex
let pp f t =
- Fmt.pf f
- "docs-ci: %s\nodoc: %s\nsherlodoc: %s"
- version (Config.odoc t.config) (Config.sherlodoc t.config)
+ Fmt.pf f "docs-ci: %s\nodoc: %s\nsherlodoc: %s" version (Config.odoc t.config)
+ (Config.sherlodoc t.config)
File "src/lib/config.mli", line 1, characters 0-0:
diff --git a/_build/default/src/lib/config.mli b/_build/default/src/lib/.formatted/config.mli
index 77b0449..5d05c31 100644
--- a/_build/default/src/lib/config.mli
+++ b/_build/default/src/lib/.formatted/config.mli
@@ -48,4 +48,4 @@ val take_n_last_versions : t -> int option
(** Number of versions to take (None for all) *)
val cache_threshold : t -> int
-(** Number of versions to keep in the cache *)
\ No newline at end of file
+(** Number of versions to keep in the cache *)
File "src/solver/solver.ml", line 1, characters 0-0:
diff --git a/_build/default/src/solver/solver.ml b/_build/default/src/solver/.formatted/solver.ml
index cd90716..956eb9f 100644
--- a/_build/default/src/solver/solver.ml
+++ b/_build/default/src/solver/.formatted/solver.ml
@@ -14,53 +14,59 @@ let env (vars : Worker.Vars.t) =
let get_names = OpamFormula.fold_left (fun a (name, _) -> name :: a) []
let universes ~packages (resolutions : OpamPackage.t list) =
-
let aux root =
- let name, version = (OpamPackage.name root, OpamPackage.version root) in
- let opamfile : OpamFile.OPAM.t =
- packages
- |> OpamPackage.Name.Map.find name
- |> OpamPackage.Version.Map.find version
- in
- let deps =
- opamfile
- |> OpamFile.OPAM.depends
- |> OpamFilter.partial_filter_formula
- (OpamFilter.deps_var_env ~build:true ~post:false ~test:false
- ~doc:false ~dev_setup:false ~dev:false)
- |> get_names
- |> OpamPackage.Name.Set.of_list
- in
- let depopts =
- opamfile
- |> OpamFile.OPAM.depopts
- |> OpamFilter.partial_filter_formula
- (OpamFilter.deps_var_env ~build:true ~post:false ~test:false
- ~doc:false ~dev_setup:false ~dev:false)
- |> get_names
- |> OpamPackage.Name.Set.of_list
- in
- let all_deps = OpamPackage.Name.Set.union deps depopts in
- let deps =
- resolutions
- |> List.filter (fun res ->
- let name = OpamPackage.name res in
- OpamPackage.Name.Set.mem name all_deps)
- in
- let result =
- OpamPackage.Set.of_list deps
- in
- result
+ let name, version = (OpamPackage.name root, OpamPackage.version root) in
+ let opamfile : OpamFile.OPAM.t =
+ packages
+ |> OpamPackage.Name.Map.find name
+ |> OpamPackage.Version.Map.find version
+ in
+ let deps =
+ opamfile
+ |> OpamFile.OPAM.depends
+ |> OpamFilter.partial_filter_formula
+ (OpamFilter.deps_var_env ~build:true ~post:false ~test:false
+ ~doc:false ~dev_setup:false ~dev:false)
+ |> get_names
+ |> OpamPackage.Name.Set.of_list
+ in
+ let depopts =
+ opamfile
+ |> OpamFile.OPAM.depopts
+ |> OpamFilter.partial_filter_formula
+ (OpamFilter.deps_var_env ~build:true ~post:false ~test:false
+ ~doc:false ~dev_setup:false ~dev:false)
+ |> get_names
+ |> OpamPackage.Name.Set.of_list
+ in
+ let all_deps = OpamPackage.Name.Set.union deps depopts in
+ let deps =
+ resolutions
+ |> List.filter (fun res ->
+ let name = OpamPackage.name res in
+ OpamPackage.Name.Set.mem name all_deps)
+ in
+ let result = OpamPackage.Set.of_list deps in
+ result
+ in
+ let simple_deps =
+ List.fold_left
+ (fun acc pkg -> OpamPackage.Map.add pkg (aux pkg) acc)
+ OpamPackage.Map.empty resolutions
in
- let simple_deps = List.fold_left (fun acc pkg -> OpamPackage.Map.add pkg (aux pkg) acc) OpamPackage.Map.empty resolutions in
-
+
let rec closure pkgs =
- let deps = List.map (fun pkg -> OpamPackage.Map.find pkg simple_deps) (OpamPackage.Set.to_list pkgs) in
+ let deps =
+ List.map
+ (fun pkg -> OpamPackage.Map.find pkg simple_deps)
+ (OpamPackage.Set.to_list pkgs)
+ in
let all = List.fold_left OpamPackage.Set.union OpamPackage.Set.empty deps in
let new_deps = OpamPackage.Set.diff all pkgs in
- if OpamPackage.Set.is_empty new_deps then pkgs else closure (OpamPackage.Set.union pkgs all)
+ if OpamPackage.Set.is_empty new_deps then pkgs
+ else closure (OpamPackage.Set.union pkgs all)
in
-
+
List.rev_map
(fun pkg ->
let name, version = (OpamPackage.name pkg, OpamPackage.version pkg) in
@@ -70,16 +76,23 @@ let universes ~packages (resolutions : OpamPackage.t list) =
|> OpamPackage.Version.Map.find version
in
let str = OpamFile.OPAM.write_to_string opamfile in
- (pkg, str, closure (OpamPackage.Map.find pkg simple_deps) |> OpamPackage.Set.elements))
- resolutions
+ ( pkg,
+ str,
+ closure (OpamPackage.Map.find pkg simple_deps)
+ |> OpamPackage.Set.elements ))
+ resolutions
-type solve_result = Worker.solve_result =
- { compile_universes : (string * string * string list) list;
- link_universes: (string * string * string list) list; } [@@deriving yojson]
+type solve_result = Worker.solve_result = {
+ compile_universes : (string * string * string list) list;
+ link_universes : (string * string * string list) list;
+}
+[@@deriving yojson]
let solve ~packages ~constraints ~root_pkgs (vars : Worker.Vars.t) =
let extended = Git_context.extend_packages packages in
- let context = Git_context.create () ~packages:extended ~env:(env vars) ~constraints in
+ let context =
+ Git_context.create () ~packages:extended ~env:(env vars) ~constraints
+ in
let t0 = Unix.gettimeofday () in
let r = Solver.solve context root_pkgs in
let t1 = Unix.gettimeofday () in
@@ -99,12 +112,12 @@ let solve ~packages ~constraints ~root_pkgs (vars : Worker.Vars.t) =
univs
in
Ok
- { compile_universes = map_universes compile_universes;
- link_universes = map_universes link_universes }
+ {
+ compile_universes = map_universes compile_universes;
+ link_universes = map_universes link_universes;
+ }
| Error diagnostics -> Error (Solver.diagnostics diagnostics)
-
-
let test commit =
Format.eprintf "Running test\n%!";
let packages =
@@ -112,27 +125,38 @@ let test commit =
( Opam_repository.open_store () >>= fun store ->
Git_context.read_packages store commit )
in
- let root_pkgs = List.map OpamPackage.Name.of_string ["ocaml";"ocaml-base-compiler";"vpnkit"] in
- let constraints =
- List.map (fun (name, rel, version) ->
- ( OpamPackage.Name.of_string name,
- (rel, OpamPackage.Version.of_string version) )) [("ocaml-base-compiler",`Geq, "4.08.0"); ("ocaml", `Leq, "5.2.0"); ("vpnkit", `Eq, "0.2.0")]
- |> OpamPackage.Name.Map.of_list
- in
- let platform = {
- Worker.Vars.arch = "x86_64";
- os="linux";
- os_distribution="linux";
- os_family="ubuntu";
- os_version="20.04";
- } in
- Format.eprintf "Calling solve\n%!";
- match solve ~packages ~constraints ~root_pkgs platform with
- | Ok packages ->
-
- Printf.printf "%s\n" (solve_result_to_yojson packages |> Yojson.Safe.to_string)
- | Error msg -> Printf.printf "%s\n" msg
-
+ let root_pkgs =
+ List.map OpamPackage.Name.of_string
+ [ "ocaml"; "ocaml-base-compiler"; "vpnkit" ]
+ in
+ let constraints =
+ List.map
+ (fun (name, rel, version) ->
+ ( OpamPackage.Name.of_string name,
+ (rel, OpamPackage.Version.of_string version) ))
+ [
+ ("ocaml-base-compiler", `Geq, "4.08.0");
+ ("ocaml", `Leq, "5.2.0");
+ ("vpnkit", `Eq, "0.2.0");
+ ]
+ |> OpamPackage.Name.Map.of_list
+ in
+ let platform =
+ {
+ Worker.Vars.arch = "x86_64";
+ os = "linux";
+ os_distribution = "linux";
+ os_family = "ubuntu";
+ os_version = "20.04";
+ }
+ in
+ Format.eprintf "Calling solve\n%!";
+ match solve ~packages ~constraints ~root_pkgs platform with
+ | Ok packages ->
+ Printf.printf "%s\n"
+ (solve_result_to_yojson packages |> Yojson.Safe.to_string)
+ | Error msg -> Printf.printf "%s\n" msg
+
let main commit =
let packages =
Lwt_main.run
File "src/lib/storage.ml", line 1, characters 0-0:
diff --git a/_build/default/src/lib/storage.ml b/_build/default/src/lib/.formatted/storage.ml
index 55903c6..9c42275 100644
--- a/_build/default/src/lib/storage.ml
+++ b/_build/default/src/lib/.formatted/storage.ml
@@ -1,16 +1,18 @@
module Base = struct
- type repository = HtmlRaw of Epoch.t | Linked of Epoch.t | Compile of Epoch.t | Prep | Prep0
+ type repository =
+ | HtmlRaw of Epoch.t
+ | Linked of Epoch.t
+ | Compile of Epoch.t
+ | Prep
+ | Prep0
let generation_folder generation =
Fpath.(v ("epoch-" ^ Epoch.digest generation))
let folder = function
- | HtmlRaw generation ->
- Fpath.(generation_folder generation / "html-raw")
- | Linked generation ->
- Fpath.(generation_folder generation / "linked")
- | Compile generation ->
- Fpath.(generation_folder generation / "compile")
+ | HtmlRaw generation -> Fpath.(generation_folder generation / "html-raw")
+ | Linked generation -> Fpath.(generation_folder generation / "linked")
+ | Compile generation -> Fpath.(generation_folder generation / "compile")
| Prep -> Fpath.v "prep"
| Prep0 -> Fpath.v "prep0"
end
@@ -50,17 +52,16 @@ let folder repository package =
Base.folder (to_base_repo repository)
// base_folder ~blessed ~prep:(repository = Prep) package)
-let cache repository package =
- folder repository package
-
+let cache repository package = folder repository package
+
let split packages =
let rec take n l =
- match n,l with
- | 0, _ -> [], l
- | n, x::xs ->
- let taken, rest = take (n-1) xs in
- x :: taken, rest
- | _, [] -> [], []
+ match (n, l) with
+ | 0, _ -> ([], l)
+ | n, x :: xs ->
+ let taken, rest = take (n - 1) xs in
+ (x :: taken, rest)
+ | _, [] -> ([], [])
in
let rec run remaining =
match take 10 remaining with
@@ -91,13 +92,15 @@ module Tar = struct
match extra_files with
| [] ->
Fmt.str
- "HASH=$((sha256sum $1/content.tar 2>/dev/null | cut -d \" \" -f 1) || echo -n \
- 'empty'); rm -f $1/content.tar; printf \"%s:$2:$HASH\\n\";"
+ "HASH=$((sha256sum $1/content.tar 2>/dev/null | cut -d \" \" -f 1) \
+ || echo -n 'empty'); rm -f $1/content.tar; printf \
+ \"%s:$2:$HASH\\n\";"
prefix
| extra_files ->
Fmt.str
"HASH=$((sha256sum $1/content.tar %s | sort | sha256sum | cut -d \" \
- \" -f 1) || echo -n 'empty'); rm $1/content.tar; printf \"%s:$2:$HASH\\n\";"
+ \" -f 1) || echo -n 'empty'); rm $1/content.tar; printf \
+ \"%s:$2:$HASH\\n\";"
(List.map (fun f -> "\"$1/" ^ f ^ "\"") extra_files
|> String.concat " ")
prefix
File "src/lib/voodoo.ml", line 1, characters 0-0:
diff --git a/_build/default/src/lib/voodoo.ml b/_build/default/src/lib/.formatted/voodoo.ml
index fead1c5..719d27c 100644
--- a/_build/default/src/lib/voodoo.ml
+++ b/_build/default/src/lib/.formatted/voodoo.ml
@@ -12,13 +12,18 @@ module Prep = struct
base
|> Spec.add
[
- run ~network ~cache "%s" @@ Misc.Cmd.list [
- "sudo apt-get update && sudo apt-get install -yy m4 pkg-config";
- Fmt.str "opam pin -ny opamh %s"
- "https://github.com/jonludlam/opamh.git#d54dbe7991af6020c36f89c300d8109664541963 && opam install -y opamh";
- "cp $(opam config var bin)/opamh /home/opam";
- "rm -rf $(opam config var prefix)"
- ]]
+ run ~network ~cache "%s"
+ @@ Misc.Cmd.list
+ [
+ "sudo apt-get update && sudo apt-get install -yy m4 \
+ pkg-config";
+ Fmt.str "opam pin -ny opamh %s"
+ "https://github.com/jonludlam/opamh.git#d54dbe7991af6020c36f89c300d8109664541963 \
+ && opam install -y opamh";
+ "cp $(opam config var bin)/opamh /home/opam";
+ "rm -rf $(opam config var prefix)";
+ ];
+ ]
end
module Odoc = struct
@@ -27,35 +32,44 @@ module Odoc = struct
base
|> Spec.add
[
- run ~network ~cache "%s" @@ Misc.Cmd.list [
- "sudo apt-get update && sudo apt-get install -yy m4";
- Fmt.str "opam pin -ny odoc-parser.dev %s && opam depext -iy odoc-parser"
- (Config.odoc config);
- Fmt.str "opam pin -ny odoc.dev %s && opam depext -iy odoc"
- (Config.odoc config);
- "cp $(opam config var bin)/odoc /home/opam";
- "rm -rf $(opam config var prefix)"
- ]]
+ run ~network ~cache "%s"
+ @@ Misc.Cmd.list
+ [
+ "sudo apt-get update && sudo apt-get install -yy m4";
+ Fmt.str
+ "opam pin -ny odoc-parser.dev %s && opam depext -iy \
+ odoc-parser"
+ (Config.odoc config);
+ Fmt.str "opam pin -ny odoc.dev %s && opam depext -iy odoc"
+ (Config.odoc config);
+ "cp $(opam config var bin)/odoc /home/opam";
+ "rm -rf $(opam config var prefix)";
+ ];
+ ]
end
module OdocDriver = struct
-
let spec ~base ~odoc_pin ~sherlodoc_pin =
let open Obuilder_spec in
base
|> Spec.add
[
- run ~network ~cache "%s" @@
- Misc.Cmd.list
- ["sudo apt-get update";
- Fmt.str "opam pin -ny sherlodoc.dev %s"
- sherlodoc_pin;
- Fmt.str "opam pin -ny odoc-parser.dev %s && opam pin -ny odoc-md.dev %s && opam pin -ny odoc.dev %s && opam pin -ny odoc-driver.dev %s"
- odoc_pin odoc_pin odoc_pin odoc_pin;
- "opam install -y odoc-md";
- "opam install -y odoc-driver sherlodoc";
- "cp $(opam config var bin)/odoc_driver_voodoo $(opam config var bin)/sherlodoc $(opam config var bin)/odoc-md /home/opam";
- "rm -rf $(opam config var prefix)"]
+ run ~network ~cache "%s"
+ @@ Misc.Cmd.list
+ [
+ "sudo apt-get update";
+ Fmt.str "opam pin -ny sherlodoc.dev %s" sherlodoc_pin;
+ Fmt.str
+ "opam pin -ny odoc-parser.dev %s && opam pin -ny \
+ odoc-md.dev %s && opam pin -ny odoc.dev %s && opam pin \
+ -ny odoc-driver.dev %s"
+ odoc_pin odoc_pin odoc_pin odoc_pin;
+ "opam install -y odoc-md";
+ "opam install -y odoc-driver sherlodoc";
+ "cp $(opam config var bin)/odoc_driver_voodoo $(opam config \
+ var bin)/sherlodoc $(opam config var bin)/odoc-md \
+ /home/opam";
+ "rm -rf $(opam config var prefix)";
+ ];
]
-
end
File "src/lib/track.ml", line 1, characters 0-0:
diff --git a/_build/default/src/lib/track.ml b/_build/default/src/lib/.formatted/track.ml
index eeb24ed..d7de969 100644
--- a/_build/default/src/lib/track.ml
+++ b/_build/default/src/lib/.formatted/track.ml
@@ -81,7 +81,8 @@ module Track = struct
let filter name =
match filter with [] -> true | lst -> List.mem (Fpath.basename name) lst
in
- Log.info (fun f -> f "Tracking packages in %a" Fpath.pp (Git.Commit.repo repo));
+ Log.info (fun f ->
+ f "Tracking packages in %a" Fpath.pp (Git.Commit.repo repo));
let* () = Current.Job.start ~level:Harmless job in
Git.with_checkout ~job repo @@ fun dir ->
let result =
File "src/solver/service.ml", line 1, characters 0-0:
diff --git a/_build/default/src/solver/service.ml b/_build/default/src/solver/.formatted/service.ml
index 82386c9..d84705a 100644
--- a/_build/default/src/solver/service.ml
+++ b/_build/default/src/solver/.formatted/service.ml
@@ -92,15 +92,20 @@ end = struct
| _ -> Fmt.failwith "BUG: bad output: %s" results)
let handle ~log request t =
- let { Worker.Solve_request.opam_repository_commit; platforms; pkgs; constraints } =
+ let {
+ Worker.Solve_request.opam_repository_commit;
+ platforms;
+ pkgs;
+ constraints;
+ } =
request
in
- Log.info log "Solving for %a, constraints %a using opam_repository_commit %s"
+ Log.info log
+ "Solving for %a, constraints %a using opam_repository_commit %s"
Fmt.(list ~sep:comma string)
pkgs
Fmt.(list ~sep:comma Worker.Solve_request.pp_constraint)
- constraints
- opam_repository_commit;
+ constraints opam_repository_commit;
let opam_repository_commit = Store.Hash.of_hex opam_repository_commit in
platforms
|> Lwt_list.map_p (fun p ->
@@ -110,7 +115,9 @@ end = struct
| Error _ as e -> Lwt.return (id, e)
| Ok packages ->
let repo_packages =
- List.map (fun (pkg, _opam, _) -> OpamPackage.of_string pkg) packages.link_universes
+ List.map
+ (fun (pkg, _opam, _) -> OpamPackage.of_string pkg)
+ packages.link_universes
in
Opam_repository.oldest_commit_with repo_packages
~from:opam_repository_commit ~log
File "src/lib/track.mli", line 1, characters 0-0:
diff --git a/_build/default/src/lib/track.mli b/_build/default/src/lib/.formatted/track.mli
index 9681d96..50774e3 100644
--- a/_build/default/src/lib/track.mli
+++ b/_build/default/src/lib/.formatted/track.mli
@@ -2,6 +2,7 @@ type t [@@deriving yojson]
val digest : t -> string
val pkg : t -> OpamPackage.t
+
val v :
limit:int option ->
filter:string list ->
File "src/solver/git_context.ml", line 1, characters 0-0:
diff --git a/_build/default/src/solver/git_context.ml b/_build/default/src/solver/.formatted/git_context.ml
index e3c302b..e5dd648 100644
--- a/_build/default/src/solver/git_context.ml
+++ b/_build/default/src/solver/.formatted/git_context.ml
@@ -28,8 +28,8 @@ let filter_deps t pkg f =
let test = OpamPackage.Name.Set.mem (OpamPackage.name pkg) t.test in
f
|> OpamFilter.partial_filter_formula (env t pkg)
- |> OpamFilter.filter_deps ~build:true ~post:true ~test ~doc:false ~dev ~dev_setup:false
- ~default:false
+ |> OpamFilter.filter_deps ~build:true ~post:true ~test ~doc:false ~dev
+ ~dev_setup:false ~default:false
let candidates t name =
match OpamPackage.Name.Map.find_opt name t.pins with
@@ -89,22 +89,34 @@ let read_package store pkg hash =
)
let extend_packages packages =
- OpamPackage.Name.Map.map (fun versions ->
- OpamPackage.Version.Map.map (fun opam ->
- let extensions = OpamFile.OPAM.extensions opam in
- let pp = OpamFormat.V.(package_formula `Conj (filtered_constraints ext_version)) in
- (try
- let extra_doc_deps = OpamStd.String.Map.find "x-extra-doc-deps" extensions in
- let raw = OpamPp.parse pp ~pos:OpamTypesBase.pos_null extra_doc_deps in
- let deps = OpamFile.OPAM.depends opam in
- let x = (OpamFormula.ands_to_list raw) in
- let y = (OpamFormula.ands_to_list deps) in
- let deps = x @ y |> OpamFormula.ands in
- let opam = OpamFile.OPAM.with_depends deps opam in
- Format.eprintf "Extended dependencies for %s:\n%!" (OpamFile.OPAM.bug_reports opam |> String.concat ", ");
- Format.eprintf "%s\n%!" (OpamFile.OPAM.write_to_string opam);
- opam
- with Not_found -> opam)) versions) packages
+ OpamPackage.Name.Map.map
+ (fun versions ->
+ OpamPackage.Version.Map.map
+ (fun opam ->
+ let extensions = OpamFile.OPAM.extensions opam in
+ let pp =
+ OpamFormat.V.(
+ package_formula `Conj (filtered_constraints ext_version))
+ in
+ try
+ let extra_doc_deps =
+ OpamStd.String.Map.find "x-extra-doc-deps" extensions
+ in
+ let raw =
+ OpamPp.parse pp ~pos:OpamTypesBase.pos_null extra_doc_deps
+ in
+ let deps = OpamFile.OPAM.depends opam in
+ let x = OpamFormula.ands_to_list raw in
+ let y = OpamFormula.ands_to_list deps in
+ let deps = x @ y |> OpamFormula.ands in
+ let opam = OpamFile.OPAM.with_depends deps opam in
+ Format.eprintf "Extended dependencies for %s:\n%!"
+ (OpamFile.OPAM.bug_reports opam |> String.concat ", ");
+ Format.eprintf "%s\n%!" (OpamFile.OPAM.write_to_string opam);
+ opam
+ with Not_found -> opam)
+ versions)
+ packages
(* Get a map of the versions inside [entry] (an entry under "packages") *)
let read_versions store (entry : Store.Value.Tree.entry) =
File "src/lib/index.ml", line 1, characters 0-0:
diff --git a/_build/default/src/lib/index.ml b/_build/default/src/lib/.formatted/index.ml
index e4e6043..7d4ba76 100644
--- a/_build/default/src/lib/index.ml
+++ b/_build/default/src/lib/.formatted/index.ml
@@ -126,8 +126,8 @@ let db =
pipeline_id = ?"
and get_pipeline_data =
Sqlite3.prepare db
- "SELECT epoch_html, epoch_linked, odoc_commit FROM docs_ci_pipeline_index \
- WHERE id = ?"
+ "SELECT epoch_html, epoch_linked, odoc_commit FROM \
+ docs_ci_pipeline_index WHERE id = ?"
in
{
@@ -164,12 +164,7 @@ let record_new_pipeline ~odoc_commit ~epoch_html ~epoch_linked =
let t = Lazy.force db in
match
Db.query_one t.record_pipeline
- Sqlite3.Data.
- [
- TEXT epoch_html;
- TEXT epoch_linked;
- TEXT odoc_commit;
- ]
+ Sqlite3.Data.[ TEXT epoch_html; TEXT epoch_linked; TEXT odoc_commit ]
with
| Sqlite3.Data.[ INT pipeline_id ] -> Ok pipeline_id
| _ -> Error "Failed to record pipeline."
@@ -251,33 +246,15 @@ let get_pipeline_data pipeline_id =
NULL;
NULL;
] ->
- ( epoch_html,
- epoch_linked,
- ""
- )
- | Sqlite3.Data.
- [
- TEXT epoch_html;
- TEXT epoch_linked;
- TEXT odoc_commit;
- ] ->
- ( epoch_html,
- epoch_linked,
- odoc_commit )
+ (epoch_html, epoch_linked, "")
+ | Sqlite3.Data.[ TEXT epoch_html; TEXT epoch_linked; TEXT odoc_commit ]
+ ->
+ (epoch_html, epoch_linked, odoc_commit)
| row -> Fmt.failwith "get_pipeline_data: invalid row %a" Db.dump_row row
in
match result with
- | [
- ( epoch_html,
- epoch_linked,
- odoc_commit );
- ] ->
- Some
- {
- epoch_html;
- epoch_linked;
- odoc_commit;
- }
+ | [ (epoch_html, epoch_linked, odoc_commit) ] ->
+ Some { epoch_html; epoch_linked; odoc_commit }
| _ -> None
(* packages - (name, version) that are failing in the latest pipeline that are passing in the latest but one *)
File "src/lib/package.mli", line 1, characters 0-0:
diff --git a/_build/default/src/lib/package.mli b/_build/default/src/lib/.formatted/package.mli
index 912665e..cbcb906 100644
--- a/_build/default/src/lib/package.mli
+++ b/_build/default/src/lib/.formatted/package.mli
@@ -22,6 +22,7 @@ and Package : sig
end
type t = Package.t
+
val to_yojson : t -> Yojson.Safe.t
val of_yojson : Yojson.Safe.t -> t Ppx_deriving_yojson_runtime.error_or
File "src/lib/package.ml", line 1, characters 0-0:
diff --git a/_build/default/src/lib/package.ml b/_build/default/src/lib/.formatted/package.ml
index 8462a4a..63d7d9a 100644
--- a/_build/default/src/lib/package.ml
+++ b/_build/default/src/lib/.formatted/package.ml
@@ -1,17 +1,17 @@
module Ocaml_version = struct
include Ocaml_version
- let to_yojson x =
- `String (Ocaml_version.to_string x)
+ let to_yojson x = `String (Ocaml_version.to_string x)
let of_yojson x =
match x with
- | `String s ->
- (match Ocaml_version.of_string s with | Ok x -> Ok x | Error (`Msg s) -> Error s)
- | _ -> Error ("Ocaml_version.of_yojson")
+ | `String s -> (
+ match Ocaml_version.of_string s with
+ | Ok x -> Ok x
+ | Error (`Msg s) -> Error s)
+ | _ -> Error "Ocaml_version.of_yojson"
end
-
module rec Universe : sig
type t [@@deriving yojson]
@@ -24,15 +24,18 @@ module rec Universe : sig
val v : Ocaml_version.t -> Package.t list -> t
val compare : t -> t -> int
end = struct
- type t = { ocaml_version : Ocaml_version.t; hash : string; deps : Package.t list; mutable extra_link_deps : Package.t list; } [@@deriving yojson]
+ type t = {
+ ocaml_version : Ocaml_version.t;
+ hash : string;
+ deps : Package.t list;
+ mutable extra_link_deps : Package.t list;
+ }
+ [@@deriving yojson]
let hash t = t.hash
let deps t = t.deps
-
let ocaml_version t = t.ocaml_version
-
let set_extra_link_deps t eld = t.extra_link_deps <- eld
-
let extra_link_deps t = t.extra_link_deps
let v ocaml_version deps =
@@ -65,7 +68,7 @@ and Package : sig
val v : Ocaml_version.t -> OpamPackage.t -> t list -> string -> t
val make :
- ocaml_version : Ocaml_version.t ->
+ ocaml_version:Ocaml_version.t ->
blacklist:string list ->
commit:string ->
root:OpamPackage.t ->
@@ -73,7 +76,12 @@ and Package : sig
(OpamPackage.t * OpamPackage.t list) list ->
t
end = struct
- type t = { ocaml_version : Ocaml_version.t; opam : O.OpamPackage.t; universe : Universe.t; commit : string }
+ type t = {
+ ocaml_version : Ocaml_version.t;
+ opam : O.OpamPackage.t;
+ universe : Universe.t;
+ commit : string;
+ }
[@@deriving yojson]
let universe t = t.universe
@@ -81,7 +89,9 @@ end = struct
let commit t = t.commit
let id t = OpamPackage.to_string t.opam ^ "-" ^ Universe.hash t.universe
let digest = id
- let v ocaml_version opam deps commit = { ocaml_version; opam; universe = Universe.v ocaml_version deps; commit }
+
+ let v ocaml_version opam deps commit =
+ { ocaml_version; opam; universe = Universe.v ocaml_version deps; commit }
let pp f { universe; opam; _ } =
Fmt.pf f "%s; %a" (OpamPackage.to_string opam) Universe.pp universe
@@ -121,47 +131,60 @@ end = struct
pkg
in
let results = obtain root in
- OpamPackage.Map.iter (fun opam_package pkg ->
- let compile_deps = List.assoc opam_package compile_deps |> OpamPackage.Set.of_list in
- let link_deps = List.assoc opam_package link_deps |> OpamPackage.Set.of_list in
- let extras = OpamPackage.Set.diff link_deps compile_deps in
- let extras =
- if OpamPackage.Set.cardinal extras = 0 then []
- else OpamPackage.Set.(elements (add opam_package extras))
- in
- let extras = List.map (fun opam_package -> obtain opam_package) extras in
- Universe.set_extra_link_deps pkg.universe extras
- ) !memo;
+ OpamPackage.Map.iter
+ (fun opam_package pkg ->
+ let compile_deps =
+ List.assoc opam_package compile_deps |> OpamPackage.Set.of_list
+ in
+ let link_deps =
+ List.assoc opam_package link_deps |> OpamPackage.Set.of_list
+ in
+ let extras = OpamPackage.Set.diff link_deps compile_deps in
+ let extras =
+ if OpamPackage.Set.cardinal extras = 0 then []
+ else OpamPackage.Set.(elements (add opam_package extras))
+ in
+ let extras =
+ List.map (fun opam_package -> obtain opam_package) extras
+ in
+ Universe.set_extra_link_deps pkg.universe extras)
+ !memo;
results
end
include Package
let topo_sort packages =
- let graph = List.map (fun pkg ->
- let universe = universe pkg in
- let deps = Universe.deps universe in
- (pkg, deps)
- ) packages
+ let graph =
+ List.map
+ (fun pkg ->
+ let universe = universe pkg in
+ let deps = Universe.deps universe in
+ (pkg, deps))
+ packages
in
let rec loop graph =
match graph with
| [] -> []
- | _ ->
- let zero_deps,others = List.partition (fun (_,x) -> x = []) graph in
- let zero_deps = List.map fst zero_deps in
- let other = List.map (fun (x,y) -> (x, List.filter (fun dep -> not (List.mem dep zero_deps)) y)) others in
- let sorted = List.sort compare zero_deps in
- sorted :: loop other
+ | _ ->
+ let zero_deps, others = List.partition (fun (_, x) -> x = []) graph in
+ let zero_deps = List.map fst zero_deps in
+ let other =
+ List.map
+ (fun (x, y) ->
+ (x, List.filter (fun dep -> not (List.mem dep zero_deps)) y))
+ others
+ in
+ let sorted = List.sort compare zero_deps in
+ sorted :: loop other
in
List.flatten (loop graph)
-
+let all_deps pkg =
+ (pkg :: (pkg |> universe |> Universe.deps))
+ @ (pkg |> universe |> Universe.extra_link_deps)
-let all_deps pkg = pkg :: (pkg |> universe |> Universe.deps) @ (pkg |> universe |> Universe.extra_link_deps)
-
-let ocaml_version pkg =
- pkg |> universe |> Universe.ocaml_version
+let ocaml_version pkg = pkg |> universe |> Universe.ocaml_version
module PackageMap = Map.Make (Package)
module PackageSet = Set.Make (Package)
@@ -240,8 +263,8 @@ module Map = PackageMap
module Set = PackageSet
let important_packages : Set.t ref = ref Set.empty
+
let add_important_packages packages =
important_packages := Set.union !important_packages packages
-let should_cache package =
- Set.mem package !important_packages
+let should_cache package = Set.mem package !important_packages
File "src/lib/config.ml", line 1, characters 0-0:
diff --git a/_build/default/src/lib/config.ml b/_build/default/src/lib/.formatted/config.ml
index c958614..abfa2bb 100644
--- a/_build/default/src/lib/config.ml
+++ b/_build/default/src/lib/.formatted/config.ml
@@ -1,11 +1,12 @@
open Cmdliner
let random =
-(* In_channel.with_open_bin "/dev/urandom" (fun
- ch ->
- let str = String.init 4 (fun _ -> In_channel.input_byte ch |> Option.get |> Char.chr) in
- let str = Base64.encode_exn str in
- str)*) "NOTRANDOM"
+ (* In_channel.with_open_bin "/dev/urandom" (fun
+ ch ->
+ let str = String.init 4 (fun _ -> In_channel.input_byte ch |> Option.get |> Char.chr) in
+ let str = Base64.encode_exn str in
+ str)*)
+ "NOTRANDOM"
module Ssh = struct
type t = {
@@ -173,11 +174,10 @@ let take_n_last_versions =
let cache_threshold =
Arg.value
- @@ Arg.opt Arg.(int) (10)
+ @@ Arg.opt Arg.(int) 10
@@ Arg.info ~doc:"Cache threshold" ~docv:"THRESHOLD" [ "cache-threshold" ]
-let v cap_file jobs track_packages
- take_n_last_versions ssh cache_threshold =
+let v cap_file jobs track_packages take_n_last_versions ssh cache_threshold =
let vat = Capnp_rpc_unix.client_only_vat () in
let cap = Capnp_rpc_unix.Cap_file.load vat cap_file |> Result.get_ok in
@@ -217,7 +217,6 @@ let odoc _ =
"https://github.com/ocaml/odoc.git#90e679061f68c5e5ee5915e280f63d842f41f300"
let sherlodoc _ = odoc ()
-
let pool _ = "docs-pipeline"
let jobs t = t.jobs
let track_packages t = t.track_packages
@@ -226,4 +225,4 @@ let ocluster_connection_do t = t.ocluster_connection_do
let ocluster_connection_prep t = t.ocluster_connection_prep
let ocluster_connection_gen t = t.ocluster_connection_gen
let ssh t = t.ssh
-let cache_threshold t = t.cache_threshold
\ No newline at end of file
+let cache_threshold t = t.cache_threshold
File "src/lib/solver.ml", line 1, characters 0-0:
diff --git a/_build/default/src/lib/solver.ml b/_build/default/src/lib/.formatted/solver.ml
index db63a0c..a727899 100644
--- a/_build/default/src/lib/solver.ml
+++ b/_build/default/src/lib/.formatted/solver.ml
@@ -74,7 +74,10 @@ let perform_constrained_solve ~solver ~pool ~job ~(platform : Platform.t) ~opam
(OpamPackage.of_string a, List.map OpamPackage.of_string b))
u
in
- Ok (solution x.packages.compile_universes, solution x.packages.link_universes, x.commit)
+ Ok
+ ( solution x.packages.compile_universes,
+ solution x.packages.link_universes,
+ x.commit )
| Ok _ -> Fmt.error_msg "??"
| Error (`Msg msg) -> Fmt.error_msg "Error from solver: %s" msg)
(fun exn ->
@@ -92,8 +95,7 @@ let perform_solve ~solver ~pool ~job ~(platform : Platform.t) ~opam track =
in
let latest = Ocaml_version.Releases.latest |> Ocaml_version.to_string in
perform_constrained_solve ~solver ~pool ~job ~platform ~opam
- (("ocaml-base-compiler", `Leq, latest)
- :: constraints)
+ (("ocaml-base-compiler", `Leq, latest) :: constraints)
let solver_version = "v2"
@@ -128,10 +130,12 @@ module Cache = struct
match value with
| Error _ -> Printf.fprintf oc "Failed"
| Ok pkg ->
- let all = Package.all_deps pkg in
- List.iter (fun pkg ->
- let opam = Package.opam pkg in
- Printf.fprintf oc "%s\n" (OpamPackage.to_string opam)) all
+ let all = Package.all_deps pkg in
+ List.iter
+ (fun pkg ->
+ let opam = Package.opam pkg in
+ Printf.fprintf oc "%s\n" (OpamPackage.to_string opam))
+ all
in
close_out oc
@@ -149,14 +153,17 @@ type key = Track.t
type t = { successes : Track.t list; failures : Track.t list }
let keys t = t.successes
+
let get key =
- Format.eprintf "Trying to 'get' %s\n%!" (Track.pkg key |> OpamPackage.to_string);
- Cache.read key |> Option.get (* is in cache ? *) |> Result.get_ok
+ Format.eprintf "Trying to 'get' %s\n%!"
+ (Track.pkg key |> OpamPackage.to_string);
+ Cache.read key |> Option.get (* is in cache ? *) |> Result.get_ok
let failures t =
t.failures
|> List.filter_map (fun k ->
- try Some (Track.pkg k, Cache.read k |> Option.get |> Result.get_error) with _ -> None)
+ try Some (Track.pkg k, Cache.read k |> Option.get |> Result.get_error)
+ with _ -> None)
(* is solved ? *)
@@ -214,22 +221,39 @@ module Solver = struct
let root = Track.pkg pkg in
let result =
match res with
- | Ok (compile_packages, link_packages, commit) ->
+ | Ok (compile_packages, link_packages, commit) -> (
Current.Job.log job "Packages returned: ";
- begin try
- let root_deps = List.assoc root compile_packages in
- Current.Job.log job "Got the root package %s" (OpamPackage.to_string root);
- let ocaml_version = List.find
+ try
+ let root_deps = List.assoc root compile_packages in
+ Current.Job.log job "Got the root package %s"
+ (OpamPackage.to_string root);
+ let ocaml_version =
+ List.find
(fun p ->
- Current.Job.log job "Checking %s" (OpamPackage.to_string p);
- List.mem (OpamPackage.name_to_string p) ["ocaml-base-compiler"; "ocaml"]) (root::root_deps) |> OpamPackage.version_to_string |> Ocaml_version.of_string_exn in
- Ok (Package.make ~ocaml_version ~blacklist ~commit ~root compile_packages link_packages)
- with Not_found ->
- Current.Job.log job "Package %s does not require OCaml" (OpamPackage.to_string root);
- let colon = Fmt.any ":" in
- Current.Job.log job "Packages returned: %a" Fmt.(list (pair ~sep:colon string (list ~sep:sp string))) (List.map (fun (a, b) -> (OpamPackage.to_string a, List.map OpamPackage.to_string b)) compile_packages);
- Error ("No OCaml dependency")
- end
+ Current.Job.log job "Checking %s"
+ (OpamPackage.to_string p);
+ List.mem
+ (OpamPackage.name_to_string p)
+ [ "ocaml-base-compiler"; "ocaml" ])
+ (root :: root_deps)
+ |> OpamPackage.version_to_string
+ |> Ocaml_version.of_string_exn
+ in
+ Ok
+ (Package.make ~ocaml_version ~blacklist ~commit ~root
+ compile_packages link_packages)
+ with Not_found ->
+ Current.Job.log job "Package %s does not require OCaml"
+ (OpamPackage.to_string root);
+ let colon = Fmt.any ":" in
+ Current.Job.log job "Packages returned: %a"
+ Fmt.(list (pair ~sep:colon string (list ~sep:sp string)))
+ (List.map
+ (fun (a, b) ->
+ ( OpamPackage.to_string a,
+ List.map OpamPackage.to_string b ))
+ compile_packages);
+ Error "No OCaml dependency")
| Error (`Msg msg) ->
Current.Job.log job "Solving failed for %s: %s"
(OpamPackage.to_string root)
File "src/ocaml_docs_ci.ml", line 1, characters 0-0:
diff --git a/_build/default/src/ocaml_docs_ci.ml b/_build/default/src/.formatted/ocaml_docs_ci.ml
index 8b03642..720e87f 100644
--- a/_build/default/src/ocaml_docs_ci.ml
+++ b/_build/default/src/.formatted/ocaml_docs_ci.ml
@@ -108,7 +108,7 @@ let main () current_config github_auth mode capnp_public_address
run_capnp capnp_public_address capnp_listen_address
>>= fun (_vat, rpc_engine_resolver) ->
let repo_opam =
- Git.clone ~schedule:hourly
+ Git.clone ~schedule:hourly
"https://github.com/ocaml/opam-repository.git"
in
let monitor = Docs_ci_lib.Monitor.make () in
File "src/lib/compile.ml", line 1, characters 0-0:
diff --git a/_build/default/src/lib/compile.ml b/_build/default/src/lib/.formatted/compile.ml
index 2396983..06b0492 100644
--- a/_build/default/src/lib/compile.ml
+++ b/_build/default/src/lib/.formatted/compile.ml
@@ -1,8 +1,11 @@
-type hashes = { compile_hash : string option; linked_hash : string option; html_hash : string option }
+type hashes = {
+ compile_hash : string option;
+ linked_hash : string option;
+ html_hash : string option;
+}
[@@deriving yojson]
type t = { package : Package.t; blessing : Package.Blessing.t; hashes : hashes }
-
type jobty = CompileAndLink | CompileOnly | LinkOnly [@@deriving yojson]
let pp_jobty =
@@ -17,56 +20,72 @@ let hashes t = t.hashes
let blessing t = t.blessing
let package t = t.package
-let spec_success ~ssh ~base ~odoc_driver_base ~odoc_pin ~sherlodoc_pin ~config ~deps ~blessing ~generation ~jobty prep =
+let spec_success ~ssh ~base ~odoc_driver_base ~odoc_pin ~sherlodoc_pin ~config
+ ~deps ~blessing ~generation ~jobty prep =
let open Obuilder_spec in
let package = Prep.package prep in
let prep_folder = Storage.folder Prep package in
let prep0_folder = Storage.folder Prep0 package in
- let compile_folder = Storage.folder (Compile (generation, blessing)) package in
+ let compile_folder =
+ Storage.folder (Compile (generation, blessing)) package
+ in
let linked_folder = Storage.folder (Linked (generation, blessing)) package in
let raw_folder = Storage.folder (HtmlRaw (generation, blessing)) package in
let opam = package |> Package.opam in
let name = opam |> OpamPackage.name_to_string in
let tools = Voodoo.Odoc.spec ~base config |> Spec.finish in
(* let compile_caches =
- deps |> List.map (fun
- {blessing; package; _} ->
- let dir = Storage.folder (Compile blessing) package in
- let fpath_strs = Fpath.segs dir in
- let cache_name = String.concat "--" fpath_strs in
- let dir = Fpath.(v "/home/opam/.cache/" // dir) in
- Obuilder_spec.Cache.v cache_name ~target:(Fpath.to_string dir)
- ) in *)
+ deps |> List.map (fun
+ {blessing; package; _} ->
+ let dir = Storage.folder (Compile blessing) package in
+ let fpath_strs = Fpath.segs dir in
+ let cache_name = String.concat "--" fpath_strs in
+ let dir = Fpath.(v "/home/opam/.cache/" // dir) in
+ Obuilder_spec.Cache.v cache_name ~target:(Fpath.to_string dir)
+ ) in *)
let common =
- (List.map (fun { blessing; package; _ } ->
- let dir = Storage.folder (Compile (generation, blessing)) package in
- Fmt.str "~/docs/docs-ci-scripts/download_compiled.sh %s %s %s %s %b" (Config.Ssh.host ssh) (Config.Ssh.storage_folder ssh) (Fpath.to_string dir) (Package.digest package ^ "-" ^ Epoch.digest generation) (Package.should_cache package)
- ) deps) @
- [Fmt.str "~/docs/docs-ci-scripts/get_prep_for_compile.sh %s %s %s %s" (Config.Ssh.host ssh) (Config.Ssh.storage_folder ssh) (Fpath.to_string prep0_folder) (Fpath.to_string prep_folder);
+ List.map
+ (fun { blessing; package; _ } ->
+ let dir = Storage.folder (Compile (generation, blessing)) package in
+ Fmt.str "~/docs/docs-ci-scripts/download_compiled.sh %s %s %s %s %b"
+ (Config.Ssh.host ssh)
+ (Config.Ssh.storage_folder ssh)
+ (Fpath.to_string dir)
+ (Package.digest package ^ "-" ^ Epoch.digest generation)
+ (Package.should_cache package))
+ deps
+ @ [
+ Fmt.str "~/docs/docs-ci-scripts/get_prep_for_compile.sh %s %s %s %s"
+ (Config.Ssh.host ssh)
+ (Config.Ssh.storage_folder ssh)
+ (Fpath.to_string prep0_folder)
+ (Fpath.to_string prep_folder);
"export OCAMLRUNPAM=b";
- ] in
+ ]
+ in
let post_compile =
[
- Misc.tar_cmd compile_folder;
- Fmt.str "time rsync -aR ./%s %s:%s/."
- (Fpath.to_string compile_folder)
- (Config.Ssh.host ssh)
- (Config.Ssh.storage_folder ssh);
- Fmt.str "set '%s'; %s"
- (Fpath.to_string compile_folder)
- (Storage.Tar.hash_command ~prefix:"COMPILE" ())]
+ Misc.tar_cmd compile_folder;
+ Fmt.str "time rsync -aR ./%s %s:%s/."
+ (Fpath.to_string compile_folder)
+ (Config.Ssh.host ssh)
+ (Config.Ssh.storage_folder ssh);
+ Fmt.str "set '%s'; %s"
+ (Fpath.to_string compile_folder)
+ (Storage.Tar.hash_command ~prefix:"COMPILE" ());
+ ]
in
let post_link_and_html =
[
- Fmt.str "mkdir -p linked && mkdir -p %a && mv linked %a/"
- Fpath.pp
+ Fmt.str "mkdir -p linked && mkdir -p %a && mv linked %a/" Fpath.pp
(Storage.Base.generation_folder generation)
Fpath.pp
(Storage.Base.generation_folder generation);
Fmt.str "mkdir -p %a" Fpath.pp linked_folder;
Misc.tar_cmd linked_folder;
Fmt.str "echo raw_folder: %s" (Fpath.to_string raw_folder);
- Fmt.str "~/docs/docs-ci-scripts/gen_status_json.sh %s" (Fpath.to_string raw_folder);
+ Fmt.str "~/docs/docs-ci-scripts/gen_status_json.sh %s"
+ (Fpath.to_string raw_folder);
Fmt.str "time rsync -aR ./%s %s:%s/."
Fpath.(to_string (parent linked_folder))
(Config.Ssh.host ssh)
@@ -86,51 +105,88 @@ let spec_success ~ssh ~base ~odoc_driver_base ~odoc_pin ~sherlodoc_pin ~config ~
Fmt.str "set '%s' raw; %s"
(Fpath.to_string raw_folder)
(Storage.hash_command ~prefix:"RAW");
- "rm -rf /home/opam/docs/*"
- ]
+ "rm -rf /home/opam/docs/*";
+ ]
in
let compile_caches =
- [Obuilder_spec.Cache.v "compile-cache" ~target:(Fmt.str "/home/opam/.cache/%a/compile" Fpath.pp (Storage.Base.generation_folder generation))] in
- let odoc_driver = Voodoo.OdocDriver.spec ~base:odoc_driver_base ~odoc_pin ~sherlodoc_pin |> Spec.finish in
+ [
+ Obuilder_spec.Cache.v "compile-cache"
+ ~target:
+ (Fmt.str "/home/opam/.cache/%a/compile" Fpath.pp
+ (Storage.Base.generation_folder generation));
+ ]
+ in
+ let odoc_driver =
+ Voodoo.OdocDriver.spec ~base:odoc_driver_base ~odoc_pin ~sherlodoc_pin
+ |> Spec.finish
+ in
base
|> Spec.children ~name:"tools" tools
|> Spec.children ~name:"odoc_driver" odoc_driver
|> Spec.add
([
- workdir "/home/opam/docs/";
- run "sudo chown opam:opam . ";
- (* Import odoc and voodoo-do *)
- copy ~from:(`Build "tools")
- [ "/home/opam/odoc";]
- ~dst:"/home/opam/";
- copy ~from:(`Build "odoc_driver")
- [ "/home/opam/odoc_driver_voodoo"; "/home/opam/sherlodoc"; "/home/opam/odoc-md"; ]
- ~dst:"/home/opam/";
- run "mv ~/sherlodoc $(opam config var bin)/sherlodoc";
- run ~network:Misc.network "sudo apt install -y jq";
- run ~network:Misc.network "git clone https://github.com/jonludlam/docs-ci-scripts.git && echo %s" Config.random;
- (* obtain the compiled dependencies, prep folder and extract it *)
- ] @ [ run ~network:Misc.network ~cache:compile_caches ~secrets:Config.Ssh.secrets "%s" @@ Misc.Cmd.list
- ((Fmt.str "ssh -MNf %s" (Config.Ssh.host ssh)) ::
- (* "rm -rf /home/opam/.cache/compile/*" :: *)
- (common @ [
- Fmt.str
- "time opam exec -- /home/opam/odoc_driver_voodoo --verbose --odoc /home/opam/odoc --odoc-md /home/opam/odoc-md --odoc-dir %a/compile --odocl-dir linked --html-dir %s --stats %s %s %s"
- Fpath.pp (Storage.Base.generation_folder generation) (Fpath.to_string (Storage.Base.folder (HtmlRaw generation)))
- name
- (match jobty with CompileAndLink -> "" | CompileOnly -> "--actions compile-only" | LinkOnly -> "--actions link-and-gen")
- (match blessing with Blessed -> "--blessed" | Universe -> "");
- (* Fmt.str "/home/opam/odoc support-files -o %s"
- (Fpath.to_string (Storage.Base.folder (HtmlRaw generation))); *)
- Fmt.str "jq . driver-benchmarks.json || true";]
- @ (match jobty with LinkOnly -> [] | _ -> post_compile)
- @ (match jobty with CompileOnly -> [] | _ -> post_link_and_html))) ])
+ workdir "/home/opam/docs/";
+ run "sudo chown opam:opam . ";
+ (* Import odoc and voodoo-do *)
+ copy ~from:(`Build "tools") [ "/home/opam/odoc" ] ~dst:"/home/opam/";
+ copy ~from:(`Build "odoc_driver")
+ [
+ "/home/opam/odoc_driver_voodoo";
+ "/home/opam/sherlodoc";
+ "/home/opam/odoc-md";
+ ]
+ ~dst:"/home/opam/";
+ run "mv ~/sherlodoc $(opam config var bin)/sherlodoc";
+ run ~network:Misc.network "sudo apt install -y jq";
+ run ~network:Misc.network
+ "git clone https://github.com/jonludlam/docs-ci-scripts.git && \
+ echo %s"
+ Config.random;
+ (* obtain the compiled dependencies, prep folder and extract it *)
+ ]
+ @ [
+ run ~network:Misc.network ~cache:compile_caches
+ ~secrets:Config.Ssh.secrets "%s"
+ @@ Misc.Cmd.list
+ (Fmt.str "ssh -MNf %s" (Config.Ssh.host ssh)
+ :: (* "rm -rf /home/opam/.cache/compile/*" :: *)
+ (common
+ @ [
+ Fmt.str
+ "time opam exec -- /home/opam/odoc_driver_voodoo \
+ --verbose --odoc /home/opam/odoc --odoc-md \
+ /home/opam/odoc-md --odoc-dir %a/compile --odocl-dir \
+ linked --html-dir %s --stats %s %s %s"
+ Fpath.pp
+ (Storage.Base.generation_folder generation)
+ (Fpath.to_string
+ (Storage.Base.folder (HtmlRaw generation)))
+ name
+ (match jobty with
+ | CompileAndLink -> ""
+ | CompileOnly -> "--actions compile-only"
+ | LinkOnly -> "--actions link-and-gen")
+ (match blessing with
+ | Blessed -> "--blessed"
+ | Universe -> "");
+ (* Fmt.str "/home/opam/odoc support-files -o %s"
+ (Fpath.to_string (Storage.Base.folder (HtmlRaw generation))); *)
+ Fmt.str "jq . driver-benchmarks.json || true";
+ ]
+ @ (match jobty with LinkOnly -> [] | _ -> post_compile)
+ @
+ match jobty with
+ | CompileOnly -> []
+ | _ -> post_link_and_html));
+ ])
let spec_failure ~ssh ~base ~config ~blessing ~generation prep =
let open Obuilder_spec in
let package = Prep.package prep in
let prep_folder = Storage.folder Prep package in
- let compile_folder = Storage.folder (Compile (generation, blessing)) package in
+ let compile_folder =
+ Storage.folder (Compile (generation, blessing)) package
+ in
let linked_folder = Storage.folder (Linked (generation, blessing)) package in
let opam = Package.opam package in
let name = OpamPackage.name_to_string opam in
@@ -144,7 +200,12 @@ let spec_failure ~ssh ~base ~config ~blessing ~generation prep =
run "sudo chown opam:opam . ";
(* Import odoc and voodoo-do *)
copy ~from:(`Build "tools")
- [ "/home/opam/odoc"; "/home/opam/odoc_driver_voodoo"; "/home/opam/sherlodoc"; "/home/opam/odoc-md" ]
+ [
+ "/home/opam/odoc";
+ "/home/opam/odoc_driver_voodoo";
+ "/home/opam/sherlodoc";
+ "/home/opam/odoc-md";
+ ]
~dst:"/home/opam/";
run "mv ~/odoc $(opam config var bin)/odoc";
run "mv ~/sherlodoc $(opam config var bin)/sherlodoc";
@@ -269,20 +330,29 @@ module Compile = struct
blessing : Package.Blessing.t;
}
- let key { config = _; odoc; sherlodoc; deps; prep; blessing; base = _; odoc_driver_base = _; jobty } =
+ let key
+ {
+ config = _;
+ odoc;
+ sherlodoc;
+ deps;
+ prep;
+ blessing;
+ base = _;
+ odoc_driver_base = _;
+ jobty;
+ } =
let odoc_sherl_hash =
- Fmt.str "%s-%s" odoc sherlodoc |> Digest.string |> Digest.to_hex in
- Fmt.str "v10-%s-%s-%s-%s-%a-%a"
- odoc_sherl_hash
+ Fmt.str "%s-%s" odoc sherlodoc |> Digest.string |> Digest.to_hex
+ in
+ Fmt.str "v10-%s-%s-%s-%s-%a-%a" odoc_sherl_hash
(Package.Blessing.to_string blessing)
(Prep.package prep |> Package.digest)
(Prep.hash prep)
Fmt.(
list (fun f { hashes = { compile_hash; _ }; _ } ->
Fmt.pf f "%a" (Fmt.option string) compile_hash))
- deps
- pp_jobty
- jobty
+ deps pp_jobty jobty
let digest t = key t |> Digest.string |> Digest.to_hex
end
@@ -293,7 +363,18 @@ module Compile = struct
let auto_cancel = true
let build { generation; _ } job
- Key.{ deps; prep; blessing; config; base; odoc_driver_base; odoc; sherlodoc; jobty } =
+ Key.
+ {
+ deps;
+ prep;
+ blessing;
+ config;
+ base;
+ odoc_driver_base;
+ odoc;
+ sherlodoc;
+ jobty;
+ } =
let open Lwt.Syntax in
let ( let** ) = Lwt_result.bind in
let package = Prep.package prep in
@@ -306,7 +387,8 @@ module Compile = struct
| Success ->
Lwt.return_ok
(spec_success ~generation ~ssh:(Config.ssh config) ~config ~base
- ~odoc_driver_base ~odoc_pin ~sherlodoc_pin ~deps ~blessing ~jobty prep)
+ ~odoc_driver_base ~odoc_pin ~sherlodoc_pin ~deps ~blessing ~jobty
+ prep)
| Failed ->
Lwt.return_ok
(spec_failure ~generation ~ssh:(Config.ssh config) ~config ~base
@@ -357,7 +439,8 @@ module Compile = struct
let compile = Option.map h compile in
let linked = Option.map h linked in
let html_raw = Option.map h html_raw in
- Lwt.return_ok { compile_hash = compile; linked_hash = linked; html_hash=html_raw }
+ Lwt.return_ok
+ { compile_hash = compile; linked_hash = linked; html_hash = html_raw }
with Invalid_argument _ ->
Lwt.return_error (`Msg "Compile: failed to parse output")
end
@@ -367,16 +450,29 @@ module CompileCache = Current_cache.Make (Compile)
let v ~generation ~config ~name ~blessing ~deps ~jobty prep =
let open Current.Syntax in
Current.component "do %s" name
- |> let> prep and> blessing and> deps
+ |> let> prep
+ and> blessing
+ and> deps
and> odoc_driver_base = Misc.default_base_image in
let package = Prep.package prep in
let base = Prep.base prep in
- (Logs.debug (fun m -> m "Prep base: %s" (Spec.to_spec base)));
+ Logs.debug (fun m -> m "Prep base: %s" (Spec.to_spec base));
let odoc = Config.odoc config in
- let sherlodoc = Config.sherlodoc config in
+ let sherlodoc = Config.sherlodoc config in
let output =
CompileCache.get { Compile.generation }
- Compile.Key.{ prep; blessing; deps; config; base; odoc_driver_base; odoc; sherlodoc; jobty }
+ Compile.Key.
+ {
+ prep;
+ blessing;
+ deps;
+ config;
+ base;
+ odoc_driver_base;
+ odoc;
+ sherlodoc;
+ jobty;
+ }
in
Current.Primitive.map_result
(Result.map (fun hashes -> { package; blessing; hashes }))
File "test/monitor/test_monitor.ml", line 1, characters 0-0:
diff --git a/_build/default/test/monitor/test_monitor.ml b/_build/default/test/monitor/.formatted/test_monitor.ml
index 087306c..b8fb883 100644
--- a/_build/default/test/monitor/test_monitor.ml
+++ b/_build/default/test/monitor/.formatted/test_monitor.ml
@@ -19,7 +19,10 @@ let running =
let fakepkg ~blessing name =
let open Docs_ci_lib in
let root = OpamPackage.of_string name in
- let pkg = Package.make ~ocaml_version:Ocaml_version.Releases.latest ~blacklist:[] ~commit:"0" ~root [] in
+ let pkg =
+ Package.make ~ocaml_version:Ocaml_version.Releases.latest ~blacklist:[]
+ ~commit:"0" ~root []
+ in
let blessing =
let set =
Package.Blessing.Set.v ~counts:(Package.Map.singleton pkg 0) [ pkg ]
File "src/lib/monitor.ml", line 1, characters 0-0:
diff --git a/_build/default/src/lib/monitor.ml b/_build/default/src/lib/.formatted/monitor.ml
index 56672bf..e103b37 100644
--- a/_build/default/src/lib/monitor.ml
+++ b/_build/default/src/lib/.formatted/monitor.ml
@@ -86,16 +86,18 @@ let render_level =
let render_list ~bullet ~level ~items ~render =
let open Tyxml_html in
- Some (ul
- (List.filter_map
- (fun (name, item) ->
- match render ~level:(level + 1) item with
- | None -> None
- | Some elt ->
- Some (li
- ~a:[ a_style ("list-style-type: " ^ bullet) ]
- [ (render_level level) [ txt name ]; elt ]))
- items))
+ Some
+ (ul
+ (List.filter_map
+ (fun (name, item) ->
+ match render ~level:(level + 1) item with
+ | None -> None
+ | Some elt ->
+ Some
+ (li
+ ~a:[ a_style ("list-style-type: " ^ bullet) ]
+ [ (render_level level) [ txt name ]; elt ]))
+ items))
let rec render ~level =
let open Tyxml_html in
@@ -113,7 +115,8 @@ let rec render ~level =
fun v -> Some (a ~a:[ a_href ("/job/" ^ job_id) ] [ txt v ])
| _ -> fun x -> Some (txt x)
with (* if current is not a primitive term *)
- | Failure _ -> (fun x -> Some (txt x))
+ | Failure _ ->
+ fun x -> Some (txt x)
in
match result with
| Error (`Msg msg) -> container ("error: " ^ msg)
@@ -136,10 +139,14 @@ let get_opam_package_info t opam_package =
in
match Package.Blessing.Set.blessed blessing_set with
| None ->
- let preps = Package.Map.filter (fun package _ ->
- Package.opam package = opam_package) t.preps in
- Or (Package.Map.bindings preps |> List.map (fun (package, tree) ->
- (Package.id package, tree)))
+ let preps =
+ Package.Map.filter
+ (fun package _ -> Package.opam package = opam_package)
+ t.preps
+ in
+ Or
+ (Package.Map.bindings preps
+ |> List.map (fun (package, tree) -> (Package.id package, tree)))
| Some blessed_package ->
let blessed_pipeline = Package.Map.find blessed_package t.trees in
blessed_pipeline
@@ -179,7 +186,11 @@ let rec to_steps description arr = function
let render_package_state t opam_package =
let name = OpamPackage.name_to_string opam_package in
- let all = Package.Map.filter_map (fun p x -> if Package.opam p = opam_package then Some x else None) t.preps in
+ let all =
+ Package.Map.filter_map
+ (fun p x -> if Package.opam p = opam_package then Some x else None)
+ t.preps
+ in
match OpamPackage.Map.find_opt opam_package t.solve_failures with
| Some reason ->
let open Tyxml_html in
@@ -193,14 +204,15 @@ let render_package_state t opam_package =
let* blessed_pipeline = get_opam_package_info t opam_package in
let open Tyxml_html in
Ok
- ([
- h1 [ txt ("Package " ^ name) ]] @
- (render ~level:1 (simplify blessed_pipeline) |> Option.to_list)
- @ [h2 [ txt ("All universes")]]
-
- @ List.flatten (List.map (fun (pkg, pipeline) ->
- ([ h3 [ txt (Package.universe pkg |> Package.Universe.hash) ] ]
- @ (render ~level:2 (simplify pipeline) |> Option.to_list))) (Package.Map.bindings all)))
+ ([ h1 [ txt ("Package " ^ name) ] ]
+ @ (render ~level:1 (simplify blessed_pipeline) |> Option.to_list)
+ @ [ h2 [ txt "All universes" ] ]
+ @ List.flatten
+ (List.map
+ (fun (pkg, pipeline) ->
+ [ h3 [ txt (Package.universe pkg |> Package.Universe.hash) ] ]
+ @ (render ~level:2 (simplify pipeline) |> Option.to_list))
+ (Package.Map.bindings all)))
let handle t ~engine:_ str =
object
@@ -270,33 +282,44 @@ let lookup_failed_compiles t =
let error_active = ref 0 in
let error_blocked = ref 0 in
let ok = ref 0 in
- let x = Package.Map.fold (fun _pkg current acc ->
- let status : step_status =
- match Current.observe current with
- | Error (`Msg msg) -> incr error_msg; Err msg
- | Error (`Active _) -> incr error_active; Active
- | Error `Blocked -> incr error_blocked; Blocked
- | Ok _ -> incr ok; OK
- in
- let container =
- try
- Current.Analysis.metadata current
- |> Current.observe
- |> Result.to_option
- |> Option.join
- |> function
- | Some { job_id = Some job_id; _ } ->
- fun v -> a ~a:[ a_href ("/job/" ^ job_id) ] [ txt v ]
- | _ -> txt
- with (* if current is not a primitive term *)
- | Failure _ -> txt
- in
- match status with
- | Err msg -> (container (Printf.sprintf "error: %s" msg)) :: acc
- | _ -> acc)
- t.html [] in
+ let x =
+ Package.Map.fold
+ (fun _pkg current acc ->
+ let status : step_status =
+ match Current.observe current with
+ | Error (`Msg msg) ->
+ incr error_msg;
+ Err msg
+ | Error (`Active _) ->
+ incr error_active;
+ Active
+ | Error `Blocked ->
+ incr error_blocked;
+ Blocked
+ | Ok _ ->
+ incr ok;
+ OK
+ in
+ let container =
+ try
+ Current.Analysis.metadata current
+ |> Current.observe
+ |> Result.to_option
+ |> Option.join
+ |> function
+ | Some { job_id = Some job_id; _ } ->
+ fun v -> a ~a:[ a_href ("/job/" ^ job_id) ] [ txt v ]
+ | _ -> txt
+ with (* if current is not a primitive term *)
+ | Failure _ -> txt
+ in
+ match status with
+ | Err msg -> container (Printf.sprintf "error: %s" msg) :: acc
+ | _ -> acc)
+ t.html []
+ in
(x, !error_active, !error_blocked, !error_msg, !ok)
-
+
let lookup_solve_failures t =
OpamPackage.Map.keys t.solve_failures |> List.map (fun k -> (k, Failed))
@@ -376,12 +399,11 @@ let render_passing_packages t =
let render_package_root t =
let max_version = map_max_versions t in
let failed, pending = lookup_failed_pending t in
- let (failed_do, n1, n2, n3, n4) = lookup_failed_compiles t in
+ let failed_do, n1, n2, n3, n4 = lookup_failed_compiles t in
let open Tyxml_html in
[
- h1 [ txt (Printf.sprintf "Failed 'do' jobs (%d, %d, %d, %d)" n1 n2 n3 n4)];
- ul (List.map (fun x ->
- li [x]) failed_do);
+ h1 [ txt (Printf.sprintf "Failed 'do' jobs (%d, %d, %d, %d)" n1 n2 n3 n4) ];
+ ul (List.map (fun x -> li [ x ]) failed_do);
h1 [ txt "Failed packages" ];
p [ txt ("total: " ^ Int.to_string (List.length failed)) ];
ul
File "src/lib/prep.ml", line 1, characters 0-0:
diff --git a/_build/default/src/lib/prep.ml b/_build/default/src/lib/.formatted/prep.ml
index 5dfd5bd..0f45f07 100644
--- a/_build/default/src/lib/prep.ml
+++ b/_build/default/src/lib/.formatted/prep.ml
@@ -15,16 +15,17 @@ module OpamPackage = struct
| _ -> Error "failed to run version"
end
-let opam_cache_version="v1"
+let opam_cache_version = "v1"
+
module Cache = struct
let fname id pkg =
let name = OpamPackage.to_string pkg in
let fname = name ^ "-opam.tar.bz2" in
- Fpath.(Current.state_dir id / name / fname )
+ Fpath.(Current.state_dir id / name / fname)
let id = "opam-files-cache-" ^ opam_cache_version
- type cache_value = (bool * string)
+ type cache_value = bool * string
let fname = fname id
@@ -34,8 +35,11 @@ module Cache = struct
| Ok true -> true
| Ok false -> false
| Error (`Msg m) ->
- Current.Job.log job "Error checking for opam files cache for package %s: %s" (OpamPackage.to_string pkg) m;
- false
+ Current.Job.log job
+ "Error checking for opam files cache for package %s: %s"
+ (OpamPackage.to_string pkg)
+ m;
+ false
let write ((pkg, value) : OpamPackage.t * cache_value) =
let fname = fname pkg in
@@ -51,111 +55,128 @@ module Cache = struct
let result = Marshal.from_channel file in
close_in file;
Some result
- with Failure _ | Sys_error _ as e ->
- Current.Job.log job "Error reading opam files cache for package %s: %s" (OpamPackage.to_string pkg) (Printexc.to_string e);
+ with (Failure _ | Sys_error _) as e ->
+ Current.Job.log job "Error reading opam files cache for package %s: %s"
+ (OpamPackage.to_string pkg)
+ (Printexc.to_string e);
None
end
-
module OpamFiles = struct
-
type t = No_context
let id = "opam-files"
-
let auto_cancel = true
module Key = struct
type t = { repo : Current_git.Commit.t; packages : OpamPackage.t list }
-
+
let digest { repo; packages } =
- Current_git.Commit.hash repo ^ String.concat ";" (List.map OpamPackage.to_string packages)
+ Current_git.Commit.hash repo
+ ^ String.concat ";" (List.map OpamPackage.to_string packages)
end
module Value = struct
- type ('a, 'b) r = ('a, 'b) result =
- | Ok of 'a
- | Error of 'b [@@deriving yojson]
+ type ('a, 'b) r = ('a, 'b) result = Ok of 'a | Error of 'b
+ [@@deriving yojson]
- type serialisable = (OpamPackage.t * ((bool * string), [`Msg of string]) r) list
- [@@deriving yojson]
+ type serialisable =
+ (OpamPackage.t * (bool * string, [ `Msg of string ]) r) list
+ [@@deriving yojson]
- type t = ((bool * string), [`Msg of string]) r OpamPackage.Map.t
+ type t = (bool * string, [ `Msg of string ]) r OpamPackage.Map.t
let to_yojson t = t |> OpamPackage.Map.bindings |> serialisable_to_yojson
- let of_yojson t = t |> serialisable_of_yojson |> Result.map OpamPackage.Map.of_list
- let marshal t = t |> to_yojson |> Yojson.Safe.to_string
- let unmarshal t = t |> Yojson.Safe.from_string |> of_yojson |> Result.get_ok
+ let of_yojson t =
+ t |> serialisable_of_yojson |> Result.map OpamPackage.Map.of_list
+
+ let marshal t = t |> to_yojson |> Yojson.Safe.to_string
+ let unmarshal t = t |> Yojson.Safe.from_string |> of_yojson |> Result.get_ok
end
let pool = Current.Pool.create ~label:"git_pool" 5
+ let build _ job { Key.repo; packages } =
+ let package_path pkg =
+ Fpath.(
+ v "packages"
+ / (OpamPackage.name pkg |> OpamPackage.Name.to_string)
+ / OpamPackage.to_string pkg)
+ in
+ let open Lwt.Infix in
+ Current.Job.start ~pool ~level:Harmless job >>= fun () ->
+ let to_do = List.filter (fun x -> not (Cache.mem job x)) packages in
+ Current_git.with_checkout ~job repo (fun path ->
+ Current.Process.with_tmpdir ~prefix:"opam-files" (fun fp ->
+ let open Lwt.Infix in
+ let r =
+ Lwt_list.map_s
+ (fun pkg ->
+ let open Lwt.Syntax in
+ let tarfile =
+ Fpath.(
+ fp / Fmt.str "%s-opam.tar.bz2" (OpamPackage.to_string pkg))
+ in
+ let has_depext =
+ match
+ Bos.OS.File.read Fpath.(path // package_path pkg / "opam")
+ with
+ | Ok content ->
+ let f = OpamFile.OPAM.read_from_string content in
+ let _x = OpamFile.OPAM.depends f in
+ OpamFile.OPAM.depexts f <> []
+ | _ -> false
+ in
+ let command =
+ Bos.Cmd.(
+ v "tar"
+ % "-C"
+ % Fpath.to_string path
+ % "-jcf"
+ % Fpath.to_string tarfile
+ % Fpath.to_string (package_path pkg))
+ in
+ Current.Job.log job "About to exec";
+ let* res =
+ Current.Process.exec ~cancellable:true ~job
+ ("", Bos.Cmd.to_list command |> Array.of_list)
+ in
+ Current.Job.log job "Finished";
+ match res with
+ | Ok () -> (
+ let res = Bos.OS.File.read tarfile in
+ match res with
+ | Ok contents ->
+ Cache.write
+ (pkg, (has_depext, Base64.encode_string contents));
+ Lwt.return (pkg, Ok ())
+ | Error (`Msg m) -> Lwt.return (pkg, Error (`Msg m)))
+ | Error (`Msg m) -> Lwt.return (pkg, Error (`Msg m)))
+ to_do
+ in
+ r >>= fun results ->
+ let results_map = OpamPackage.Map.of_list results in
+ List.map
+ (fun x ->
+ match Cache.read job x with
+ | Some v -> (x, Ok v)
+ | None -> (
+ match OpamPackage.Map.find x results_map with
+ | Error (`Msg m) -> (x, Error (`Msg m))
+ | Ok () ->
+ (x, Error (`Msg "Failed to read cache after writing"))
+ | exception Not_found ->
+ (x, Error (`Msg "Didn't even try to solve for package"))
+ ))
+ packages
+ |> OpamPackage.Map.of_list
+ |> Lwt.return_ok))
-
-let build _ job { Key.repo; packages } =
- let package_path pkg =
- Fpath.(v "packages" / (OpamPackage.name pkg |> OpamPackage.Name.to_string) / OpamPackage.to_string pkg)
- in
- let open Lwt.Infix in
- Current.Job.start ~pool ~level:Harmless job >>= fun () ->
- let to_do = List.filter (fun x -> not (Cache.mem job x)) packages in
- Current_git.with_checkout ~job repo (fun path ->
- Current.Process.with_tmpdir ~prefix:"opam-files" (fun fp ->
- let open Lwt.Infix in
- let r = Lwt_list.map_s (fun pkg ->
- let open Lwt.Syntax in
- let tarfile = Fpath.(fp / (Fmt.str "%s-opam.tar.bz2" (OpamPackage.to_string pkg))) in
- let has_depext =
- match Bos.OS.File.read Fpath.(path // package_path pkg / "opam") with
- | Ok content ->
- let f = OpamFile.OPAM.read_from_string content in
- let _x = OpamFile.OPAM.depends f in
- OpamFile.OPAM.depexts f <> []
- | _ -> false
- in
- let command =
- Bos.Cmd.(
- v "tar"
- % "-C"
- % Fpath.to_string path
- % "-jcf"
- % Fpath.to_string tarfile
- % (Fpath.to_string (package_path pkg) )
- )
- in
- Current.Job.log job "About to exec";
- let* res = Current.Process.exec ~cancellable:true ~job
- ("", Bos.Cmd.to_list command |> Array.of_list) in
- Current.Job.log job "Finished";
- match res with
- | Ok () -> begin
- let res = Bos.OS.File.read tarfile in
- match res with
- | Ok contents ->
- Cache.write (pkg, (has_depext, Base64.encode_string contents));
- Lwt.return (pkg, Ok ())
- | Error (`Msg m) -> Lwt.return (pkg, Error (`Msg m))
- end
- | Error (`Msg m) -> Lwt.return (pkg, Error (`Msg m))
- ) to_do in
- r >>= fun results ->
- let results_map = OpamPackage.Map.of_list results in
- List.map (fun x ->
- match Cache.read job x with
- | Some v -> (x, Ok v)
- | None -> begin
- match OpamPackage.Map.find x results_map with
- | Error (`Msg m) -> (x, Error (`Msg m))
- | Ok () -> (x, Error (`Msg "Failed to read cache after writing"))
- | exception Not_found ->
- (x, Error (`Msg "Didn't even try to solve for package"))
- end) packages |> OpamPackage.Map.of_list |> Lwt.return_ok
- ))
let pp f { Key.repo; packages } =
Fmt.pf f "opamfiles\n%a\n%a" Current_git.Commit.pp_short repo
- (Fmt.list (Fmt.of_to_string OpamPackage.to_string)) packages
-
+ (Fmt.list (Fmt.of_to_string OpamPackage.to_string))
+ packages
end
module type F = Current_cache.S.BUILDER
@@ -177,41 +198,50 @@ let not_base x =
])
let add_base ocaml_version init =
- let add_one x pkgs =
- if List.mem x pkgs then pkgs else x :: pkgs
- in
+ let add_one x pkgs = if List.mem x pkgs then pkgs else x :: pkgs in
let mk n v =
OpamPackage.create
(OpamPackage.Name.of_string n)
(OpamPackage.Version.of_string v)
in
let v = ocaml_version in
- let std = List.fold_right add_one [
- mk "conf-graphviz" "0.1";
- mk "conf-which" "1";
- mk "base-unix" "base";
- mk "base-bigarray" "base";
- mk "base-threads" "base";
- mk "ocaml-base-compiler" (Ocaml_version.to_string ocaml_version);
- mk "ocaml" (Ocaml_version.to_string ocaml_version);
- ] init in
- let extra = List.assoc (Ocaml_version.major v) [
- 5, [mk "base-domains" "base";
- mk "base-nnp" "base";
- mk "base-effects" "base";
- mk "host-arch-x86_64" "1";
- mk "host-system-other" "1";
- mk "ocaml-options-vanilla" "1";
- ];
- 4, [
- mk "ocaml-options-vanilla" "1";
- mk "host-system-other" "1";
- mk "host-arch-x86_64" "1";
- ]
-
- ] in
+ let std =
+ List.fold_right add_one
+ [
+ mk "conf-graphviz" "0.1";
+ mk "conf-which" "1";
+ mk "base-unix" "base";
+ mk "base-bigarray" "base";
+ mk "base-threads" "base";
+ mk "ocaml-base-compiler" (Ocaml_version.to_string ocaml_version);
+ mk "ocaml" (Ocaml_version.to_string ocaml_version);
+ ]
+ init
+ in
+ let extra =
+ List.assoc (Ocaml_version.major v)
+ [
+ ( 5,
+ [
+ mk "base-domains" "base";
+ mk "base-nnp" "base";
+ mk "base-effects" "base";
+ mk "host-arch-x86_64" "1";
+ mk "host-system-other" "1";
+ mk "ocaml-options-vanilla" "1";
+ ] );
+ ( 4,
+ [
+ mk "ocaml-options-vanilla" "1";
+ mk "host-system-other" "1";
+ mk "host-arch-x86_64" "1";
+ ] );
+ ]
+ in
let ocaml_config =
- if Ocaml_version.major v = 5 then [ mk "ocaml-config" "3"] else if Ocaml_version.minor v >= 12 then [ mk "ocaml-config" "2"] else [ mk "ocaml-config" "1" ]
+ if Ocaml_version.major v = 5 then [ mk "ocaml-config" "3" ]
+ else if Ocaml_version.minor v >= 12 then [ mk "ocaml-config" "2" ]
+ else [ mk "ocaml-config" "1" ]
in
ocaml_config @ std @ extra
@@ -232,12 +262,13 @@ let spec ~ssh ~tools_base ~base ~opamfiles (prep : Package.t) =
let ocaml_version = Package.ocaml_version prep in
let prep_caches =
- [Obuilder_spec.Cache.v "prep-cache" ~target:"/home/opam/.cache/prep"] in
+ [ Obuilder_spec.Cache.v "prep-cache" ~target:"/home/opam/.cache/prep" ]
+ in
let cache = cache @ prep_caches in
let packages_topo_list =
- all_deps
+ all_deps
|> Package.topo_sort
|> List.filter (fun x -> Package.opam x |> not_base)
in
@@ -253,19 +284,17 @@ let spec ~ssh ~tools_base ~base ~opamfiles (prep : Package.t) =
### output ###
# Error: link: /home/opam/.cache/dune/db/v2/temp/promoting: Invalid cross-device link
*)
-
-
let prep_storage_folder = (Storage.Prep0, prep) in
let prep_folder = Storage.folder Prep0 prep in
let create_dir_and_copy_logs_if_not_exist =
let command =
Fmt.str
- "([ -d $1 ] || (echo FAILED:$2 && mkdir -p $1 && cp ~/opam.err.log \
- $1 && opam show $3 --raw > $1/opam)) && (%s)"
+ "([ -d $1 ] || (echo FAILED:$2 && mkdir -p $1 && cp ~/opam.err.log $1 \
+ && opam show $3 --raw > $1/opam)) && (%s)"
(Misc.tar_cmd (Fpath.v "$1"))
in
- Storage.for_all [prep_storage_folder] command
+ Storage.for_all [ prep_storage_folder ] command
in
let install_opamfiles pkg tar_b64 =
@@ -273,79 +302,111 @@ let spec ~ssh ~tools_base ~base ~opamfiles (prep : Package.t) =
Fmt.str "echo %s ; echo %s | base64 -d | sudo tar -jxC /src" name tar_b64
in
- let install_all_opamfiles = List.filter_map (fun pkg ->
- match OpamPackage.Map.find (Package.opam pkg) opamfiles with
- | Ok (_, opamfiles) ->
- Some (install_opamfiles (Package.opam pkg) opamfiles)
- | Error (`Msg _m) ->
- None
- | exception Not_found ->
- None) packages_topo_list
+ let install_all_opamfiles =
+ List.filter_map
+ (fun pkg ->
+ match OpamPackage.Map.find (Package.opam pkg) opamfiles with
+ | Ok (_, opamfiles) ->
+ Some (install_opamfiles (Package.opam pkg) opamfiles)
+ | Error (`Msg _m) -> None
+ | exception Not_found -> None)
+ packages_topo_list
in
- let install_cmds = List.map (run ~network "%s") (Misc.Cmd.list_list install_all_opamfiles) in
+ let install_cmds =
+ List.map (run ~network "%s") (Misc.Cmd.list_list install_all_opamfiles)
+ in
let install_packages =
- "echo \"START OF INSTALL PACKAGES\" && date" :: (
- List.flatten @@ List.map (fun pkg ->
- match OpamPackage.Map.find (Package.opam pkg) opamfiles with
- | Ok (_has_depext, _opamfiles) ->
- [ Fmt.str "time ~/docs/docs-ci-scripts/download_prep.sh %s %s %s %s"
- (Config.Ssh.host ssh)
- (Config.Ssh.storage_folder ssh)
- (Fpath.to_string (Storage.folder Storage.Prep0 pkg))
- (if Package.should_cache pkg then "true" else "false")
- ]
- | Error _ | exception _ ->
- [ Fmt.str "echo Failed to find opamfiles for %s"
- (Package.opam pkg |> OpamPackage.to_string)
- ]) packages_topo_list)
+ "echo \"START OF INSTALL PACKAGES\" && date"
+ :: (List.flatten
+ @@ List.map
+ (fun pkg ->
+ match OpamPackage.Map.find (Package.opam pkg) opamfiles with
+ | Ok (_has_depext, _opamfiles) ->
+ [
+ Fmt.str
+ "time ~/docs/docs-ci-scripts/download_prep.sh %s %s %s %s"
+ (Config.Ssh.host ssh)
+ (Config.Ssh.storage_folder ssh)
+ (Fpath.to_string (Storage.folder Storage.Prep0 pkg))
+ (if Package.should_cache pkg then "true" else "false");
+ ]
+ | Error _ | (exception _) ->
+ [
+ Fmt.str "echo Failed to find opamfiles for %s"
+ (Package.opam pkg |> OpamPackage.to_string);
+ ])
+ packages_topo_list)
in
- let any_depexts = List.exists (fun pkg -> match OpamPackage.Map.find (Package.opam pkg) opamfiles with | Ok (has_depext, _) -> has_depext | Error _ | exception _ -> false) packages_topo_list in
+ let any_depexts =
+ List.exists
+ (fun pkg ->
+ match OpamPackage.Map.find (Package.opam pkg) opamfiles with
+ | Ok (has_depext, _) -> has_depext
+ | Error _ | (exception _) -> false)
+ packages_topo_list
+ in
let extra_opamfiles =
- List.flatten (List.map (fun pkg ->
- match OpamPackage.Map.find pkg opamfiles with
- | Ok (_has_depext, opamfiles) ->
- [ install_opamfiles pkg opamfiles ]
- | _ | exception _ -> [ Fmt.str "echo Missing %s" (OpamPackage.to_string pkg)]) (add_base ocaml_version [])
- )
+ List.flatten
+ (List.map
+ (fun pkg ->
+ match OpamPackage.Map.find pkg opamfiles with
+ | Ok (_has_depext, opamfiles) -> [ install_opamfiles pkg opamfiles ]
+ | _ | (exception _) ->
+ [ Fmt.str "echo Missing %s" (OpamPackage.to_string pkg) ])
+ (add_base ocaml_version []))
in
let pkg_opamfile =
- (match OpamPackage.Map.find (Package.opam prep) opamfiles with
- | Ok (_has_depext, opamfiles) ->
+ match OpamPackage.Map.find (Package.opam prep) opamfiles with
+ | Ok (_has_depext, opamfiles) ->
[ install_opamfiles (Package.opam prep) opamfiles ]
- | Error _ | exception _ -> [])
+ | Error _ | (exception _) -> []
in
let post_steps =
- [ "echo \"START OF BUILD PACKAGE\" && date";
-
- "opam update && /home/opam/opamh make-state --output $(opam var prefix)/.opam-switch/switch-state";
- if any_depexts then Fmt.str "opam depext %s" (OpamPackage.to_string (Package.opam prep)) else "echo no depexts";
- Fmt.str "(opam update && opam install -vv --debug-level=2 --confirm-level=unsafe-yes --solver=builtin-0install %s 2>&1 && opam clean -s | tee ~/opam.err.log) || echo \
- 'Failed to install all packages'" (Package.opam prep |> OpamPackage.to_string);
+ [
+ "echo \"START OF BUILD PACKAGE\" && date";
+ "opam update && /home/opam/opamh make-state --output $(opam var \
+ prefix)/.opam-switch/switch-state";
+ (if any_depexts then
+ Fmt.str "opam depext %s" (OpamPackage.to_string (Package.opam prep))
+ else "echo no depexts");
+ Fmt.str
+ "(opam update && opam install -vv --debug-level=2 \
+ --confirm-level=unsafe-yes --solver=builtin-0install %s 2>&1 && opam \
+ clean -s | tee ~/opam.err.log) || echo 'Failed to install all \
+ packages'"
+ (Package.opam prep |> OpamPackage.to_string);
"echo \"END OF BUILD PACKAGE\" && date";
Fmt.str "mkdir -p %s" (Fpath.to_string prep_folder);
- Fmt.str "/home/opam/opamh save --output=%s/content.tar %s" (Fpath.to_string prep_folder) (Package.opam prep |> OpamPackage.name |> OpamPackage.Name.to_string);
+ Fmt.str "/home/opam/opamh save --output=%s/content.tar %s"
+ (Fpath.to_string prep_folder)
+ (Package.opam prep |> OpamPackage.name |> OpamPackage.Name.to_string);
Fmt.str "time rsync -aR ./%s %s:%s/."
(Fpath.to_string prep_folder)
(Config.Ssh.host ssh)
(Config.Ssh.storage_folder ssh);
"rm -f /tmp/*.tar";
"rm -rf $(opam var prefix)";
- ] in
+ ]
+ in
- let persistent_ssh = [ Fmt.str "/usr/bin/time -f \"SSH time %%E\" ssh -MNf %s" (Config.Ssh.host ssh) ] in
+ let persistent_ssh =
+ [
+ Fmt.str "/usr/bin/time -f \"SSH time %%E\" ssh -MNf %s"
+ (Config.Ssh.host ssh);
+ ]
+ in
let copy_results =
- (create_dir_and_copy_logs_if_not_exist)
- @
- (* Compute hashes *)
- (Storage.for_all [prep_stoage_folder]
- (Storage.Tar.hash_command ~prefix:"HASHES" ()))
+ create_dir_and_copy_logs_if_not_exist
+ @ (* Compute hashes *)
+ Storage.for_all [ prep_storage_folder ]
+ (Storage.Tar.hash_command ~prefix:"HASHES" ())
in
(* let install_packages = persistent_ssh @ pkg_opamfile @ extra_opamfiles @ install_packages @ post_steps @ copy_results in *)
@@ -354,39 +415,53 @@ let spec ~ssh ~tools_base ~base ~opamfiles (prep : Package.t) =
|> Spec.children ~name:"tools" tools
|> Spec.add
([
- (* Install required packages *)
- run "sudo mkdir /src";
- run "echo b3BhbS12ZXJzaW9uOiAiMi4wIgpicm93c2U6ICJodHRwczovL29wYW0ub2NhbWwub3JnL3BrZy8iCnVwc3RyZWFtOiAiaHR0cHM6Ly9naXRodWIuY29tL29jYW1sL29wYW0tcmVwb3NpdG9yeS90cmVlL21hc3Rlci8iCmFubm91bmNlOiBbCiIiIgpbV0FSTklOR10gb3BhbSBpcyBvdXQtb2YtZGF0ZS4gUGxlYXNlIGNvbnNpZGVyIHVwZGF0aW5nIGl0IChodHRwczovL29wYW0ub2NhbWwub3JnL2RvYy9JbnN0YWxsLmh0bWwpCiIiIiB7KG9wYW0tdmVyc2lvbiA+PSAiMi4xLjB+fiIgJiBvcGFtLXZlcnNpb24gPCAiMi4xLjUiKSB8IG9wYW0tdmVyc2lvbiA8ICIyLjAuMTAifQoiIiIKW0lORk9dIG9wYW0gMi4xIGluY2x1ZGVzIG1hbnkgcGVyZm9ybWFuY2UgaW1wcm92ZW1lbnRzIG92ZXIgMi4wOyBwbGVhc2UgY29uc2lkZXIgdXBncmFkaW5nIChodHRwczovL29wYW0ub2NhbWwub3JnL2RvYy9JbnN0YWxsLmh0bWwpCiIiIiB7b3BhbS12ZXJzaW9uID49ICIyLjAuMTAiICYgb3BhbS12ZXJzaW9uIDwgIjIuMS4wfn4ifQpdCg== | base64 -d | sudo tee /src/repo";
- run "ls -lR /src";
- (* Re-initialise opam after switching from opam.2.0 to 2.3. *)
- run ~network
- "sudo ln -f /usr/bin/opam-2.3 /usr/bin/opam && opam repo remove default --all && opam init --reinit \
- -ni";
- run "sudo mkdir /src/packages";
- run "opam repo add opam /src";
- copy ~from:(`Build "tools")
- [ "/home/opam/opamh" ]
- ~dst:"/home/opam/";
- (* Enable build cache conditionally on dune version *)
- env "DUNE_CACHE" "disabled";
- run "mkdir /home/opam/docs";
- (* Pre-install some of the most popular packages *)
- run ~network:Misc.network "sudo apt-get update && sudo apt-get install -qq -yy pkg-config libgmp-dev libev-dev libssl-dev zlib1g-dev libpcre3-dev libffi-dev m4 xdot autoconf libsqlite3-dev cmake libcurl4-gnutls-dev libpcre2-dev libsdl2-dev time python3 libexpat1-dev libcairo2-dev";
-
- run ~network:Misc.network "git clone https://github.com/jonludlam/docs-ci-scripts.git /home/opam/docs/docs-ci-scripts && echo %s" Config.random;
-
- (* run ~network ~cache ~secrets:Config.Ssh.secrets "%s" @@ Misc.Cmd.list install_packages *)
- ] @ install_cmds @ [
- run ~network ~cache ~secrets:Config.Ssh.secrets "%s" @@ Misc.Cmd.list (persistent_ssh @ pkg_opamfile @ extra_opamfiles @ install_packages) ] @ [
- run ~network ~cache ~secrets:Config.Ssh.secrets "%s" @@ Misc.Cmd.list (post_steps @ copy_results);
- ]
- )
-
-
+ (* Install required packages *)
+ run "sudo mkdir /src";
+ run
+ "echo \
+ b3BhbS12ZXJzaW9uOiAiMi4wIgpicm93c2U6ICJodHRwczovL29wYW0ub2NhbWwub3JnL3BrZy8iCnVwc3RyZWFtOiAiaHR0cHM6Ly9naXRodWIuY29tL29jYW1sL29wYW0tcmVwb3NpdG9yeS90cmVlL21hc3Rlci8iCmFubm91bmNlOiBbCiIiIgpbV0FSTklOR10gb3BhbSBpcyBvdXQtb2YtZGF0ZS4gUGxlYXNlIGNvbnNpZGVyIHVwZGF0aW5nIGl0IChodHRwczovL29wYW0ub2NhbWwub3JnL2RvYy9JbnN0YWxsLmh0bWwpCiIiIiB7KG9wYW0tdmVyc2lvbiA+PSAiMi4xLjB+fiIgJiBvcGFtLXZlcnNpb24gPCAiMi4xLjUiKSB8IG9wYW0tdmVyc2lvbiA8ICIyLjAuMTAifQoiIiIKW0lORk9dIG9wYW0gMi4xIGluY2x1ZGVzIG1hbnkgcGVyZm9ybWFuY2UgaW1wcm92ZW1lbnRzIG92ZXIgMi4wOyBwbGVhc2UgY29uc2lkZXIgdXBncmFkaW5nIChodHRwczovL29wYW0ub2NhbWwub3JnL2RvYy9JbnN0YWxsLmh0bWwpCiIiIiB7b3BhbS12ZXJzaW9uID49ICIyLjAuMTAiICYgb3BhbS12ZXJzaW9uIDwgIjIuMS4wfn4ifQpdCg== \
+ | base64 -d | sudo tee /src/repo";
+ run "ls -lR /src";
+ (* Re-initialise opam after switching from opam.2.0 to 2.3. *)
+ run ~network
+ "sudo ln -f /usr/bin/opam-2.3 /usr/bin/opam && opam repo remove \
+ default --all && opam init --reinit -ni";
+ run "sudo mkdir /src/packages";
+ run "opam repo add opam /src";
+ copy ~from:(`Build "tools") [ "/home/opam/opamh" ] ~dst:"/home/opam/";
+ (* Enable build cache conditionally on dune version *)
+ env "DUNE_CACHE" "disabled";
+ run "mkdir /home/opam/docs";
+ (* Pre-install some of the most popular packages *)
+ run ~network:Misc.network
+ "sudo apt-get update && sudo apt-get install -qq -yy pkg-config \
+ libgmp-dev libev-dev libssl-dev zlib1g-dev libpcre3-dev \
+ libffi-dev m4 xdot autoconf libsqlite3-dev cmake \
+ libcurl4-gnutls-dev libpcre2-dev libsdl2-dev time python3 \
+ libexpat1-dev libcairo2-dev";
+ run ~network:Misc.network
+ "git clone https://github.com/jonludlam/docs-ci-scripts.git \
+ /home/opam/docs/docs-ci-scripts && echo %s"
+ Config.random;
+ (* run ~network ~cache ~secrets:Config.Ssh.secrets "%s" @@ Misc.Cmd.list install_packages *)
+ ]
+ @ install_cmds
+ @ [
+ run ~network ~cache ~secrets:Config.Ssh.secrets "%s"
+ @@ Misc.Cmd.list
+ (persistent_ssh
+ @ pkg_opamfile
+ @ extra_opamfiles
+ @ install_packages);
+ ]
+ @ [
+ run ~network ~cache ~secrets:Config.Ssh.secrets "%s"
+ @@ Misc.Cmd.list (post_steps @ copy_results);
+ ])
+
type prep_result = Success | Failed
-type prep_output =
- {
+type prep_output = {
base : Spec.t;
hash : string;
prep_hash : string;
@@ -400,10 +475,7 @@ module Prep = struct
let id = "voodoo-prep2"
let auto_cancel = true
-
module Key = struct
-
-
type t = {
prep : Package.t;
base : Spec.t;
@@ -412,14 +484,13 @@ module Prep = struct
opamfiles : (bool * string) Current.or_error OpamPackage.Map.t;
}
- let digest { prep; base = _; tools_base = _; config = _; opamfiles = _; } =
+ let digest { prep; base = _; tools_base = _; config = _; opamfiles = _ } =
(* base is derived from 'prep' so we don't need to include it in the hash *)
Fmt.str "%s\n%s\n%s\n" prep_version (Package.digest prep)
(Package.digest prep)
end
- let pp f Key.{ prep; _ } =
- Fmt.pf f "Voodoo prep %a" Package.pp prep
+ let pp f Key.{ prep; _ } = Fmt.pf f "Voodoo prep %a" Package.pp prep
module Value = struct
type item = Storage.id_hash [@@deriving yojson]
@@ -429,8 +500,7 @@ module Prep = struct
let unmarshal t = t |> Yojson.Safe.from_string |> of_yojson |> Result.get_ok
end
- let build No_context job Key.{ prep; base; tools_base; config; opamfiles; }
- =
+ let build No_context job Key.{ prep; base; tools_base; config; opamfiles } =
let open Lwt.Syntax in
let ( let** ) = Lwt_result.bind in
(* Problem: no rebuild if the opam definition changes without affecting the universe hash.
@@ -438,7 +508,9 @@ module Prep = struct
requires changes in the solver.
For now we rebuild only if voodoo-prep changes.
*)
- let spec = spec ~ssh:(Config.ssh config) ~base ~tools_base ~opamfiles prep in
+ let spec =
+ spec ~ssh:(Config.ssh config) ~base ~tools_base ~opamfiles prep
+ in
let action = Misc.to_ocluster_submission spec in
let version = Misc.cache_hint prep in
Current.Job.log job "Prep job: prep %a" Package.pp prep;
@@ -461,8 +533,7 @@ module Prep = struct
--connect ocluster-submission.cap --cache-hint %s \\@.--secret \
ssh_privkey:id_rsa --secret ssh_pubkey:id_rsa.pub--secret \
ssh_config:ssh_config@.@."
- (Spec.to_spec spec)
- cache_hint);
+ (Spec.to_spec spec) cache_hint);
Capnp_rpc_lwt.Capability.with_ref build_job @@ fun build_job ->
(* extract result from logs *)
@@ -521,7 +592,6 @@ end
module PrepCache = Current_cache.Make (Prep)
-
type t = prep_output
let hash t = t.hash
@@ -545,20 +615,29 @@ module StringSet = Set.Make (String)
let weekly = Current_cache.Schedule.v ~valid_for:(Duration.of_day 7) ()
let v ~config ~spec ~deps ~opamfiles ~prep =
- let open Current.Syntax in
+ let open Current.Syntax in
let opamfiles : OpamFiles.Value.t Current.t = opamfiles in
Current.component "voodoo-prep %s" (prep |> Package.digest)
- |> let> spec and> opamfiles and> deps
- and> tools_base = Misc.default_base_image in
- ignore deps;
- PrepCache.get No_context { prep; config; base = spec; tools_base; opamfiles; }
- |> Current.Primitive.map_result (Result.map (fun (artifacts_branches_output, _failed) ->
- let _artifacts_branches_output =
- artifacts_branches_output
- |> List.to_seq
- |> Seq.map (fun Storage.{ id; hash } -> (id, hash))
- |> StringMap.of_seq
- in
-
- { base = spec; prep_hash = ""; result=Success; package=prep; hash="" }))
-
+ |> let> spec
+ and> opamfiles
+ and> deps
+ and> tools_base = Misc.default_base_image in
+ ignore deps;
+ PrepCache.get No_context
+ { prep; config; base = spec; tools_base; opamfiles }
+ |> Current.Primitive.map_result
+ (Result.map (fun (artifacts_branches_output, _failed) ->
+ let _artifacts_branches_output =
+ artifacts_branches_output
+ |> List.to_seq
+ |> Seq.map (fun Storage.{ id; hash } -> (id, hash))
+ |> StringMap.of_seq
+ in
+
+ {
+ base = spec;
+ prep_hash = "";
+ result = Success;
+ package = prep;
+ hash = "";
+ }))
File "src/pipelines/docs.ml", line 1, characters 0-0:
diff --git a/_build/default/src/pipelines/docs.ml b/_build/default/src/pipelines/.formatted/docs.ml
index afd53e5..bcd14e3 100644
--- a/_build/default/src/pipelines/docs.ml
+++ b/_build/default/src/pipelines/.formatted/docs.ml
@@ -11,7 +11,6 @@ end
module OpamPackageNameCurrentMap = CurrentMap (OpamPackage.Name.Map)
module OpamPackageVersionCurrentMap = CurrentMap (OpamPackage.Version.Map)
-
let status = function
| Error (`Msg msg) -> Monitor.Err msg
| Error (`Active _) -> Active
@@ -51,8 +50,7 @@ let rec summarise description arr = function
items
|> current_list_flatten
-
-let count_refs (all:Package.Set.t) =
+let count_refs (all : Package.Set.t) =
let prep_jobs : int ref Package.Map.t ref = ref Package.Map.empty in
let rec get_prep_job package =
@@ -60,168 +58,169 @@ let count_refs (all:Package.Set.t) =
let ref = Package.Map.find package !prep_jobs in
incr ref
with Not_found ->
- let dependencies =
- Package.universe package |> Package.Universe.deps
- in
- let () =
- List.iter
- (fun p ->
- get_prep_job p)
- dependencies
- in
+ let dependencies = Package.universe package |> Package.Universe.deps in
+ let () = List.iter (fun p -> get_prep_job p) dependencies in
prep_jobs := Package.Map.add package (ref 1) !prep_jobs
in
Package.Set.iter (fun x -> get_prep_job x) all;
!prep_jobs
-type 'a compile_job =
- { job: 'a Current.t;
- monitor : Monitor.pipeline_tree;
- }
-
+type 'a compile_job = { job : 'a Current.t; monitor : Monitor.pipeline_tree }
+
let compile ~generation ~config
~(blessed : Package.Blessing.Set.t Current.t OpamPackage.Map.t)
- (preps : (Prep.t compile_job Package.Map.t)) =
- let compilation_jobs : Compile.t compile_job option Package.Map.t ref = ref Package.Map.empty in
- let link_jobs : Compile.t compile_job option Package.Map.t ref = ref Package.Map.empty in
+ (preps : Prep.t compile_job Package.Map.t) =
+ let compilation_jobs : Compile.t compile_job option Package.Map.t ref =
+ ref Package.Map.empty
+ in
+ let link_jobs : Compile.t compile_job option Package.Map.t ref =
+ ref Package.Map.empty
+ in
let rec get_compilation_job link package =
let name = package |> Package.opam |> OpamPackage.to_string in
let extra_deps =
Package.universe package |> Package.Universe.extra_link_deps
- in
+ in
let job_cache = if link then link_jobs else compilation_jobs in
try Package.Map.find package !job_cache
- with Not_found ->
+ with Not_found -> (
let job =
Package.Map.find_opt package preps
- |> Option.map @@ fun { job=prep; _ } ->
+ |> Option.map @@ fun { job = prep; _ } ->
let dependencies =
Package.universe package |> Package.Universe.deps
in
- begin
- if List.length extra_deps > 0 then
- Logs.info (fun m -> m "Extra link deps for %s: %s (link: %b)"
- name
- (extra_deps |> List.map (fun x -> x |> Package.opam |> OpamPackage.to_string) |> String.concat ", ") link)
- end;
+ if List.length extra_deps > 0 then
+ Logs.info (fun m ->
+ m "Extra link deps for %s: %s (link: %b)" name
+ (extra_deps
+ |> List.map (fun x ->
+ x |> Package.opam |> OpamPackage.to_string)
+ |> String.concat ", ")
+ link);
let compile_dependencies_names =
List.filter_map
(fun p ->
- get_compilation_job false p |> Option.map (fun { job; _ } -> (p, job)))
+ get_compilation_job false p
+ |> Option.map (fun { job; _ } -> (p, job)))
dependencies
in
let link_dependencies_names =
- if link then
- List.filter_map
- (fun p ->
- get_compilation_job false p |> Option.map (fun { job; _ } -> (p, job)))
- extra_deps
- else
- []
- in
+ if link then
+ List.filter_map
+ (fun p ->
+ get_compilation_job false p
+ |> Option.map (fun { job; _ } -> (p, job)))
+ extra_deps
+ else []
+ in
let compile_dependencies =
compile_dependencies_names |> List.map snd
in
- let link_dependencies =
- link_dependencies_names |> List.map snd
- in
+ let link_dependencies = link_dependencies_names |> List.map snd in
let blessing =
OpamPackage.Map.find (Package.opam package) blessed
|> Current.map (fun b -> Package.Blessing.Set.get b package)
in
- let deps, jobty = match List.length extra_deps, link with
- | 0, _ -> (Current.list_seq compile_dependencies), Compile.CompileAndLink
+ let deps, jobty =
+ match (List.length extra_deps, link) with
+ | 0, _ ->
+ (Current.list_seq compile_dependencies, Compile.CompileAndLink)
| _, false ->
- Logs.info (fun m -> m "Creating CompileOnly job");
- (Current.list_seq compile_dependencies), Compile.CompileOnly
+ Logs.info (fun m -> m "Creating CompileOnly job");
+ (Current.list_seq compile_dependencies, Compile.CompileOnly)
| _, true ->
- Logs.info (fun m -> m "Creating LinkOnly job");
- (Current.list_seq (compile_dependencies @ link_dependencies)), Compile.LinkOnly
+ Logs.info (fun m -> m "Creating LinkOnly job");
+ ( Current.list_seq (compile_dependencies @ link_dependencies),
+ Compile.LinkOnly )
in
let node =
- Compile.v ~generation ~config ~name ~blessing
- ~deps
- ~jobty
- prep
+ Compile.v ~generation ~config ~name ~blessing ~deps ~jobty prep
in
let monitor =
- Monitor.(
- Seq
- [
- ( "do-deps",
- Item prep);
-
- ("do-compile", Item node);
- ])
+ Monitor.(Seq [ ("do-deps", Item prep); ("do-compile", Item node) ])
in
- (jobty, {job=node; monitor; })
+ (jobty, { job = node; monitor })
in
match job with
| None -> None
| Some (CompileAndLink, job) ->
- compilation_jobs := Package.Map.add package (Some job) !compilation_jobs;
- link_jobs := Package.Map.add package (Some job) !link_jobs;
- Some job
+ compilation_jobs :=
+ Package.Map.add package (Some job) !compilation_jobs;
+ link_jobs := Package.Map.add package (Some job) !link_jobs;
+ Some job
| Some (CompileOnly, job) ->
- compilation_jobs := Package.Map.add package (Some job) !compilation_jobs;
- Some job
+ compilation_jobs :=
+ Package.Map.add package (Some job) !compilation_jobs;
+ Some job
| Some (LinkOnly, job) ->
- link_jobs := Package.Map.add package (Some job) !link_jobs;
- Some job
+ link_jobs := Package.Map.add package (Some job) !link_jobs;
+ Some job)
in
- let get_compilation_node package _ =
- get_compilation_job true package
+ let get_compilation_node package _ = get_compilation_job true package in
+ let compile_jobs =
+ Package.Map.filter_map get_compilation_node preps |> Package.Map.bindings
in
- let compile_jobs = Package.Map.filter_map get_compilation_node preps |> Package.Map.bindings in
compile_jobs
-let prep ~config
- ~opamfiles (all:Package.Set.t) =
- let prep_jobs : Prep.t compile_job Package.Map.t ref = ref Package.Map.empty in
+let prep ~config ~opamfiles (all : Package.Set.t) =
+ let prep_jobs : Prep.t compile_job Package.Map.t ref =
+ ref Package.Map.empty
+ in
let rec get_prep_job package =
try Package.Map.find package !prep_jobs
with Not_found ->
let job =
- let dependencies =
- Package.universe package |> Package.Universe.deps
+ let dependencies = Package.universe package |> Package.Universe.deps in
+ let dep_opams =
+ List.map Package.opam dependencies |> OpamPackage.Set.of_list
+ in
+ let opams =
+ OpamPackage.Set.add (Package.opam package) dep_opams
+ |> OpamPackage.Set.to_list
in
- let dep_opams = List.map Package.opam dependencies |> OpamPackage.Set.of_list in
- let opams = OpamPackage.Set.add (Package.opam package) dep_opams |> OpamPackage.Set.to_list in
let ocaml_version = Package.ocaml_version package in
let all_opams = Prep.add_base ocaml_version opams in
-
+
let prep_dependencies_names =
List.map
- (fun p ->
- get_prep_job p |> (fun {job; _} -> (p, job)))
+ (fun p -> get_prep_job p |> fun { job; _ } -> (p, job))
dependencies
in
- let prep_dependencies =
- prep_dependencies_names |> List.map snd
- in
+ let prep_dependencies = prep_dependencies_names |> List.map snd in
let base_image = Misc.get_base_image package in
- let opamfiles = Current.map (fun x ->
- List.filter_map (fun p -> try Some (p, OpamPackage.Map.find p x) with _ -> None) all_opams |> OpamPackage.Map.of_list) opamfiles in
+ let opamfiles =
+ Current.map
+ (fun x ->
+ List.filter_map
+ (fun p ->
+ try Some (p, OpamPackage.Map.find p x) with _ -> None)
+ all_opams
+ |> OpamPackage.Map.of_list)
+ opamfiles
+ in
let node =
Prep.v ~config
- ~deps:(Current.list_seq prep_dependencies)
- ~spec:base_image ~opamfiles ~prep:package
+ ~deps:(Current.list_seq prep_dependencies)
+ ~spec:base_image ~opamfiles ~prep:package
in
let monitor =
Monitor.(
Seq
[
( "prep",
- And (("prep "^Package.id package, Item node)
- :: List.map (fun (pkg, compile) ->
- ("prep dependency "^Package.id pkg, Item compile)) prep_dependencies_names));
- ]
- )
+ And
+ (("prep " ^ Package.id package, Item node)
+ :: List.map
+ (fun (pkg, compile) ->
+ ("prep dependency " ^ Package.id pkg, Item compile))
+ prep_dependencies_names) );
+ ])
in
- { job=node; monitor; }
+ { job = node; monitor }
in
prep_jobs := Package.Map.add package job !prep_jobs;
@@ -230,24 +229,24 @@ let prep ~config
let get_prep_node package =
get_prep_job package
(* |> Option.map @@ fun (compile, monitor) ->
- let node =
- Html.v ~generation ~config
- ~name:(package |> Package.opam |> OpamPackage.to_string)
- ~voodoo:voodoo_gen compile
- in
- let monitor =
- match monitor with
- | Monitor.Seq lst -> Monitor.Seq (lst @ [ ("do-html", Item node) ])
- | _ -> assert false
- in
- let package_status = Monitor.pipeline_state monitor in
- let _index =
- (* let+ step_list = summarise "" [] monitor *)
- (* DEBUGGING THE MEMORY BLOAT -- Skip the summarising for now *)
- let+ pipeline_id in
- Index.record package pipeline_id package_status []
- in
- (node, monitor) *)
+ let node =
+ Html.v ~generation ~config
+ ~name:(package |> Package.opam |> OpamPackage.to_string)
+ ~voodoo:voodoo_gen compile
+ in
+ let monitor =
+ match monitor with
+ | Monitor.Seq lst -> Monitor.Seq (lst @ [ ("do-html", Item node) ])
+ | _ -> assert false
+ in
+ let package_status = Monitor.pipeline_state monitor in
+ let _index =
+ (* let+ step_list = summarise "" [] monitor *)
+ (* DEBUGGING THE MEMORY BLOAT -- Skip the summarising for now *)
+ let+ pipeline_id in
+ Index.record package pipeline_id package_status []
+ in
+ (node, monitor) *)
in
Package.Set.iter (fun x -> ignore (get_prep_node x)) all;
!prep_jobs
@@ -338,9 +337,7 @@ let v ~config ~opam ~monitor ~migrations () =
| Some path -> Index.migrate path
| None -> Current.return ()
in
- let generation =
- Epoch.v config
- in
+ let generation = Epoch.v config in
(* 0) Housekeeping - run migrations *)
let* _ = migrations in
Log.info (fun f -> f "0) Migrations");
@@ -359,7 +356,9 @@ let v ~config ~opam ~monitor ~migrations () =
Log.info (fun f -> f "2) Solver result");
(* 3.a) From solver results, obtain a list of package.version.universe corresponding to prep jobs *)
let all_packages_jobs =
- solver_result |> Solver.keys |> List.filter_map (fun x -> try Some (Solver.get x) with _ -> None)
+ solver_result
+ |> Solver.keys
+ |> List.filter_map (fun x -> try Some (Solver.get x) with _ -> None)
in
Log.info (fun f -> f "2.5) Solver result...");
(* 3.b) Expand that list to all the obtainable package.version.universe *)
@@ -368,14 +367,19 @@ let v ~config ~opam ~monitor ~migrations () =
all_packages_jobs
|> List.rev_map Package.all_deps
|> List.flatten
- |> List.filter (fun pkg -> Ocaml_version.compare (Package.ocaml_version pkg) (Ocaml_version.Releases.v4_04_2) >= 0)
+ |> List.filter (fun pkg ->
+ Ocaml_version.compare
+ (Package.ocaml_version pkg)
+ Ocaml_version.Releases.v4_04_2
+ >= 0)
|> Package.Set.of_list
in
- Log.info (fun f -> f "3) All packages (%d)" (Package.Set.cardinal all_packages));
+ Log.info (fun f ->
+ f "3) All packages (%d)" (Package.Set.cardinal all_packages));
+
(* 4) Schedule a somewhat small set of jobs to obtain at least one universe for each package.version *)
(* 4a) Decide on a docker tag for each job *)
(* 5) Run the preparation step *)
-
let weekly = Current_cache.Schedule.v ~valid_for:(Duration.of_day 7) () in
let repo_opam =
@@ -383,24 +387,37 @@ let v ~config ~opam ~monitor ~migrations () =
"https://github.com/ocaml/opam-repository.git"
in
- let opamfiles =
- Current.component "opamfiles" |>
+ let opamfiles =
+ Current.component "opamfiles"
+ |>
let> repo_opam in
- let packages = Package.Set.to_list all_packages |> List.map Package.opam |> OpamPackage.Set.of_list in
- let extra = [
- "conf-graphviz.0.1";
- "conf-which.1";
- "base-threads.base";
- "ocaml-options-vanilla.1";
- "base-bigarray.base";
- "base-domains.base";
- "base-nnp.base";
- "base-effects.base";
- "host-arch-x86_64.1";
- "host-system-other.1";
- ] in
- let packages = OpamPackage.Set.union (List.map OpamPackage.of_string extra |> OpamPackage.Set.of_list) packages |> OpamPackage.Set.to_list in
- Prep.OpamFilesCache.get No_context Prep.OpamFiles.Key.{ repo = repo_opam; packages }
+ let packages =
+ Package.Set.to_list all_packages
+ |> List.map Package.opam
+ |> OpamPackage.Set.of_list
+ in
+ let extra =
+ [
+ "conf-graphviz.0.1";
+ "conf-which.1";
+ "base-threads.base";
+ "ocaml-options-vanilla.1";
+ "base-bigarray.base";
+ "base-domains.base";
+ "base-nnp.base";
+ "base-effects.base";
+ "host-arch-x86_64.1";
+ "host-system-other.1";
+ ]
+ in
+ let packages =
+ OpamPackage.Set.union
+ (List.map OpamPackage.of_string extra |> OpamPackage.Set.of_list)
+ packages
+ |> OpamPackage.Set.to_list
+ in
+ Prep.OpamFilesCache.get No_context
+ Prep.OpamFiles.Key.{ repo = repo_opam; packages }
in
let counts = count_refs all_packages in
@@ -412,45 +429,46 @@ let v ~config ~opam ~monitor ~migrations () =
let counts = List.sort (fun (_, x) (_, y) -> compare !x !y) counts in
- let (_, hd),tl = List.hd counts, List.tl counts in
- let (last_count, last_val, count) = List.fold_left (fun (cur_count,cur_val,all) (_, count) ->
- if cur_val = !count
- then (cur_count + 1, cur_val, all)
- else (1, !count, (cur_val, cur_count) :: all)) (1, !hd, []) tl in
+ let (_, hd), tl = (List.hd counts, List.tl counts) in
+ let last_count, last_val, count =
+ List.fold_left
+ (fun (cur_count, cur_val, all) (_, count) ->
+ if cur_val = !count then (cur_count + 1, cur_val, all)
+ else (1, !count, (cur_val, cur_count) :: all))
+ (1, !hd, []) tl
+ in
let all_counts = (last_val, last_count) :: count in
- List.iter (fun (v, count) ->
- Printf.printf "%d %d\n%!" v count) all_counts;
-
- let cache_packages = List.fold_left (fun acc (v, count) ->
- if !count > Config.cache_threshold config
- then Package.Set.add v acc
- else acc) Package.Set.empty counts
+ List.iter (fun (v, count) -> Printf.printf "%d %d\n%!" v count) all_counts;
+
+ let cache_packages =
+ List.fold_left
+ (fun acc (v, count) ->
+ if !count > Config.cache_threshold config then Package.Set.add v acc
+ else acc)
+ Package.Set.empty counts
in
Package.add_important_packages cache_packages;
Log.info (fun f ->
- f "4) Cache packages (%d)" (Package.Set.cardinal cache_packages));
+ f "4) Cache packages (%d)" (Package.Set.cardinal cache_packages));
(* let cumulative =
- let x = ref 0 in
- List.map (fun (v, count) ->
- let y = !x + count in
- x := y;
- (v, y)) all_counts in
+ let x = ref 0 in
+ List.map (fun (v, count) ->
+ let y = !x + count in
+ x := y;
+ (v, y)) all_counts in
- List.iter (fun (v, count) ->
- Printf.printf "%d %d\n%!" v count) cumulative;
-
- ignore @@ exit 0; *)
+ List.iter (fun (v, count) ->
+ Printf.printf "%d %d\n%!" v count) cumulative;
+ ignore @@ exit 0; *)
let prepped' : Prep.t compile_job Package.Map.t =
- prep ~config
- ~opamfiles all_packages
+ prep ~config ~opamfiles all_packages
in
- Log.info (fun f ->
- f ".. %d prepped nodes" (Package.Map.cardinal prepped'));
+ Log.info (fun f -> f ".. %d prepped nodes" (Package.Map.cardinal prepped'));
Log.info (fun f -> f "5) Prep nodes");
(* 6) Promote packages to the main tree *)
@@ -470,7 +488,7 @@ let v ~config ~opam ~monitor ~migrations () =
in
let by_opam_package =
Package.Map.fold
- (fun package {job=prep; _ } opam_map ->
+ (fun package { job = prep; _ } opam_map ->
let opam = Package.opam package in
let job =
let+ job = Current.state ~hidden:true prep in
@@ -497,29 +515,32 @@ let v ~config ~opam ~monitor ~migrations () =
(* 7) Odoc compile and html-generate artifacts *)
let html, html_input_node, package_pipeline_tree, html2 =
- let compile_monitor =
- compile ~generation ~config ~blessed
- prepped'
+ let compile_monitor = compile ~generation ~config ~blessed prepped' in
+ let html2 =
+ List.map
+ (fun (pkg, { job = compile; _ }) -> (pkg, compile))
+ compile_monitor
in
- let html2 = List.map (fun (pkg, { job=compile; _ }) -> (pkg, compile)) compile_monitor in
Log.info (fun f ->
f ".. %d compilation nodes" (List.length compile_monitor));
let c, compile_node =
compile_monitor
- |> List.map (fun (a, {job=b; _}) -> (a, b))
+ |> List.map (fun (a, { job = b; _ }) -> (a, b))
|> compile_hierarchical_collapse ~input:solver_result_c
in
( c |> List.to_seq |> Package.Map.of_seq,
compile_node,
compile_monitor
- |> List.map (fun (a, { monitor=b; _}) -> (a, b))
+ |> List.map (fun (a, { monitor = b; _ }) -> (a, b))
|> List.to_seq
|> Package.Map.of_seq,
- html2 |> Package.Map.of_list)
+ html2 |> Package.Map.of_list )
in
Log.info (fun f -> f "7) Odoc compile nodes");
- let prep_pipeline_tree = Package.Map.map (fun (p : Prep.t compile_job) -> p.monitor) prepped' in
+ let prep_pipeline_tree =
+ Package.Map.map (fun (p : Prep.t compile_job) -> p.monitor) prepped'
+ in
(* 7.b) Inform the monitor *)
let () =
dune build @fmt failed
"/usr/bin/env" "bash" "-c" "opam exec -- dune build @fmt --ignore-promoted-rules || (echo "dune build @fmt failed"; exit 2)" failed with exit status 2
2025-04-22 12:40.13: Job failed: Failed: Build failed