2025-04-22 12:36.31: New job: test ocurrent/ocaml-docs-ci https://github.com/ocurrent/ocaml-docs-ci.git#refs/heads/next-gen2 (cb095a367a8964bd594109f331419be2af01f996) (linux-x86_64:(lint-fmt)) Base: ocaml/opam:debian-12-ocaml-4.08@sha256:23c85daf4eb9a91d154cf2f9fed47f4f9b9fa788adc9c1fc7f0802e3dc06d7c4 ocamlformat version: version 0.26.0 (from opam) To reproduce locally: git clone --recursive "https://github.com/ocurrent/ocaml-docs-ci.git" -b "next-gen2" && cd "ocaml-docs-ci" && git reset --hard cb095a36 cat > Dockerfile <<'END-OF-DOCKERFILE' FROM ocaml/opam:debian-12-ocaml-4.08@sha256:23c85daf4eb9a91d154cf2f9fed47f4f9b9fa788adc9c1fc7f0802e3dc06d7c4 USER 1000:1000 RUN cd ~/opam-repository && (git cat-file -e 1b767795a4e028a4d45f916f1598c24ee7a7b6cb || git fetch origin master) && git reset -q --hard 1b767795a4e028a4d45f916f1598c24ee7a7b6cb && git log --no-decorate -n1 --oneline && opam update -u RUN opam depext -i dune WORKDIR /src RUN opam depext -i ocamlformat=0.26.0 COPY --chown=1000:1000 . /src/ RUN opam exec -- dune build @fmt --ignore-promoted-rules || (echo "dune build @fmt failed"; exit 2) END-OF-DOCKERFILE docker build . END-REPRO-BLOCK 2025-04-22 12:36.31: Using cache hint "ocurrent/ocaml-docs-ci-ocaml/opam:debian-12-ocaml-4.08@sha256:23c85daf4eb9a91d154cf2f9fed47f4f9b9fa788adc9c1fc7f0802e3dc06d7c4-debian-12-4.08_opam-2.3-ocamlformat-1b767795a4e028a4d45f916f1598c24ee7a7b6cb" 2025-04-22 12:36.31: Using OBuilder spec: ((from ocaml/opam:debian-12-ocaml-4.08@sha256:23c85daf4eb9a91d154cf2f9fed47f4f9b9fa788adc9c1fc7f0802e3dc06d7c4) (user (uid 1000) (gid 1000)) (run (cache (opam-archives (target /home/opam/.opam/download-cache))) (network host) (shell "cd ~/opam-repository && (git cat-file -e 1b767795a4e028a4d45f916f1598c24ee7a7b6cb || git fetch origin master) && git reset -q --hard 1b767795a4e028a4d45f916f1598c24ee7a7b6cb && git log --no-decorate -n1 --oneline && opam update -u")) (run (cache (opam-archives (target /home/opam/.opam/download-cache))) (network host) (shell "opam depext -i dune")) (workdir /src) (run (cache (opam-archives (target /home/opam/.opam/download-cache))) (network host) (shell "opam depext -i ocamlformat=0.26.0")) (copy (src .) (dst /src/)) (run (shell "opam exec -- dune build @fmt --ignore-promoted-rules || (echo \"dune build @fmt failed\"; exit 2)")) ) 2025-04-22 12:36.31: Waiting for resource in pool OCluster 2025-04-22 12:36.31: Waiting for worker… 2025-04-22 12:36.32: Got resource from pool OCluster Building on clete.caelum.ci.dev All commits already cached HEAD is now at cb095a3 Remove blocked items from monitor page (from ocaml/opam:debian-12-ocaml-4.08@sha256:23c85daf4eb9a91d154cf2f9fed47f4f9b9fa788adc9c1fc7f0802e3dc06d7c4) 2025-04-22 12:36.33 ---> using "44ab722c6d76cc4aa083f99994b6eb04bd97004720a37e6316ca14ec9a9ee4aa" from cache /: (user (uid 1000) (gid 1000)) /: (run (cache (opam-archives (target /home/opam/.opam/download-cache))) (network host) (shell "cd ~/opam-repository && (git cat-file -e 1b767795a4e028a4d45f916f1598c24ee7a7b6cb || git fetch origin master) && git reset -q --hard 1b767795a4e028a4d45f916f1598c24ee7a7b6cb && git log --no-decorate -n1 --oneline && opam update -u")) 1b767795a4 Merge pull request #27704 from maiste/release-dune-3.18.0 <><> Updating package repositories ><><><><><><><><><><><><><><><><><><><><><><> [default] Initialised default (at git+file:///home/opam/opam-repository): [INFO] opam 2.1 and 2.2 include many performance and security improvements over 2.0; please consider upgrading (https://opam.ocaml.org/doc/Install.html) Everything as up-to-date as possible (run with --verbose to show unavailable upgrades). However, you may "opam upgrade" these packages explicitly, which will ask permission to downgrade or uninstall the conflicting packages. Nothing to do. # Run eval $(opam env) to update the current shell environment 2025-04-22 12:37.45 ---> saved as "5af033ca5998041d246dc8498882e9dc146ac54b140907f2e1916f6f582b7193" /: (run (cache (opam-archives (target /home/opam/.opam/download-cache))) (network host) (shell "opam depext -i dune")) # Detecting depexts using vars: arch=x86_64, os=linux, os-distribution=debian, os-family=debian # No extra OS packages requirements found. # All required OS packages found. # Now letting opam install the packages The following actions will be performed: - install dune 3.18.0 <><> Gathering sources ><><><><><><><><><><><><><><><><><><><><><><><><><><><><> [dune.3.18.0] found in cache <><> Processing actions <><><><><><><><><><><><><><><><><><><><><><><><><><><><> -> installed dune.3.18.0 Done. # Run eval $(opam env) to update the current shell environment 2025-04-22 12:38.50 ---> saved as "f3ca3dfc78385ac1381397f690e815b0455920d345180c72e1fc56947dfe3be3" /: (workdir /src) /src: (run (cache (opam-archives (target /home/opam/.opam/download-cache))) (network host) (shell "opam depext -i ocamlformat=0.26.0")) # Detecting depexts using vars: arch=x86_64, os=linux, os-distribution=debian, os-family=debian # No extra OS packages requirements found. # All required OS packages found. # Now letting opam install the packages The following actions will be performed: - install sexplib0 v0.14.0 [required by base] - install cmdliner 1.3.0 [required by ocamlformat] - install dune-build-info 3.18.0 [required by ocamlformat-lib] - install menhirLib 20240715 [required by ocamlformat-lib] - install menhirCST 20240715 [required by menhir] - install ocamlbuild 0.16.1 [required by fpath, astring, uuseg] - install menhirSdk 20240715 [required by ocamlformat-lib] - install either 1.0.0 [required by ocamlformat-lib] - install result 1.5 [required by ocamlformat-lib] - install ocaml-version 4.0.0 [required by ocamlformat-lib] - install camlp-streams 5.0.1 [required by ocamlformat-lib] - install csexp 1.5.2 [required by ocamlformat-lib] - install seq base [required by re] - install ocamlfind 1.9.8 [required by ocp-indent, astring, fpath, uuseg] - install fix 20230505 [required by ocamlformat-lib] - install menhir 20240715 [required by ocamlformat-lib] - install dune-configurator 3.18.0 [required by base] - install re 1.11.0 [required by ocamlformat] - install topkg 1.0.8 [required by fpath, astring, uuseg] - install base-bytes base [required by ocp-indent] - install base v0.14.3 [required by ocamlformat-lib] - install uutf 1.0.4 [required by ocamlformat-lib] - install astring 0.8.5 [required by ocamlformat-lib] - install ocp-indent 1.8.1 [required by ocamlformat-lib] - install stdio v0.14.0 [required by ocamlformat-lib] - install uucp 15.0.0 [required by uuseg] - install fpath 0.7.3 [required by ocamlformat-lib] - install uuseg 15.0.0 [required by ocamlformat-lib] - install ocamlformat-lib 0.26.0 [required by ocamlformat] - install ocamlformat 0.26.0 ===== 30 to install ===== <><> Gathering sources ><><><><><><><><><><><><><><><><><><><><><><><><><><><><> [astring.0.8.5] found in cache [base.v0.14.3] found in cache [camlp-streams.5.0.1] found in cache [cmdliner.1.3.0] found in cache [csexp.1.5.2] found in cache [dune-build-info.3.18.0] found in cache [dune-configurator.3.18.0] found in cache [either.1.0.0] found in cache [fix.20230505] found in cache [fpath.0.7.3] found in cache [menhir.20240715] found in cache [menhirCST.20240715] found in cache [menhirLib.20240715] found in cache [menhirSdk.20240715] found in cache [ocaml-version.4.0.0] found in cache [ocamlbuild.0.16.1] found in cache [ocamlfind.1.9.8] found in cache [ocamlformat.0.26.0] found in cache [ocamlformat-lib.0.26.0] found in cache [ocp-indent.1.8.1] found in cache [re.1.11.0] found in cache [result.1.5] found in cache [sexplib0.v0.14.0] found in cache [stdio.v0.14.0] found in cache [topkg.1.0.8] found in cache [uucp.15.0.0] found in cache [uuseg.15.0.0] found in cache [uutf.1.0.4] found in cache <><> Processing actions <><><><><><><><><><><><><><><><><><><><><><><><><><><><> -> installed seq.base -> installed camlp-streams.5.0.1 -> installed csexp.1.5.2 -> installed either.1.0.0 -> installed fix.20230505 -> installed cmdliner.1.3.0 -> installed menhirCST.20240715 -> installed menhirLib.20240715 -> installed menhirSdk.20240715 -> installed ocaml-version.4.0.0 -> installed re.1.11.0 -> installed result.1.5 -> installed sexplib0.v0.14.0 -> installed dune-build-info.3.18.0 -> installed dune-configurator.3.18.0 -> installed ocamlfind.1.9.8 -> installed base-bytes.base -> installed ocp-indent.1.8.1 -> installed ocamlbuild.0.16.1 -> installed base.v0.14.3 -> installed stdio.v0.14.0 -> installed topkg.1.0.8 -> installed uutf.1.0.4 -> installed astring.0.8.5 -> installed fpath.0.7.3 -> installed menhir.20240715 -> installed uucp.15.0.0 -> installed uuseg.15.0.0 -> installed ocamlformat-lib.0.26.0 -> installed ocamlformat.0.26.0 Done. <><> ocp-indent.1.8.1 installed successfully ><><><><><><><><><><><><><><><><><> => This package requires additional configuration for use in editors. Install package 'user-setup', or manually: * for Emacs, add these lines to ~/.emacs: (add-to-list 'load-path "/home/opam/.opam/4.08/share/emacs/site-lisp") (require 'ocp-indent) * for Vim, add this line to ~/.vimrc: set rtp^="/home/opam/.opam/4.08/share/ocp-indent/vim" # Run eval $(opam env) to update the current shell environment 2025-04-22 12:40.09 ---> saved as "9d4e05d7215de4c09734aa004b02c994d19dec2363b796d29ddf14afeabf8120" /src: (copy (src .) (dst /src/)) 2025-04-22 12:40.09 ---> saved as "088799107dc9135160b6966064e80be37c3da6a0350bf603b7afaba8b1a84690" /src: (run (shell "opam exec -- dune build @fmt --ignore-promoted-rules || (echo \"dune build @fmt failed\"; exit 2)")) ocamlformat: ignoring "src/solver/solver.mli" (misplaced documentation comments - warning 50) File "src/solver/solver.mli", line 6, characters 0-69: 6 | (** [test hash] runs a test with commit [hash] in opam-repository. *) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Warning 50 [unexpected-docstring]: ambiguous documentation comment Hint: (Warning 50) This file contains a documentation comment (** ... *) that the OCaml compiler does not know how to attach to the AST. OCamlformat does not support these cases. You can find more information at: https://github.com/ocaml-ppx/ocamlformat#overview. If you'd like to disable this check and let ocamlformat make a choice (though it might not be consistent with the ocaml compilers and odoc), you can set the --no-comment-check option. -> required by _build/default/src/solver/.formatted/solver.mli -> required by alias src/solver/.formatted/fmt -> required by alias src/solver/fmt File "src/api/solver/worker.ml", line 1, characters 0-0: diff --git a/_build/default/src/api/solver/worker.ml b/_build/default/src/api/solver/.formatted/worker.ml index 4782905..9b6163a 100644 --- a/_build/default/src/api/solver/worker.ml +++ b/_build/default/src/api/solver/.formatted/worker.ml @@ -12,9 +12,11 @@ module Vars = struct [@@deriving yojson] end -type solve_result = - { compile_universes : (string * string * string list) list; - link_universes: (string * string * string list) list; } [@@deriving yojson, ord] +type solve_result = { + compile_universes : (string * string * string list) list; + link_universes : (string * string * string list) list; +} +[@@deriving yojson, ord] (** A set of packages for a single build. *) module Selection = struct @@ -40,7 +42,15 @@ module Solve_request = struct [@@deriving yojson] let pp_constraint ppf (pkg, rel, vsn) = - Format.fprintf ppf "%s%s%s" pkg (match rel with | `Eq -> "=" | `Geq -> ">=" | `Gt -> ">" | `Leq -> "<=" | `Lt -> "<" | `Neq -> "<>") vsn + Format.fprintf ppf "%s%s%s" pkg + (match rel with + | `Eq -> "=" + | `Geq -> ">=" + | `Gt -> ">" + | `Leq -> "<=" + | `Lt -> "<" + | `Neq -> "<>") + vsn end (** The response from the solver. *) File "src/solver/main.ml", line 1, characters 0-0: diff --git a/_build/default/src/solver/main.ml b/_build/default/src/solver/.formatted/main.ml index 52e95bb..05bda32 100644 --- a/_build/default/src/solver/main.ml +++ b/_build/default/src/solver/.formatted/main.ml @@ -76,8 +76,9 @@ let () = export service ~on:Lwt_unix.stdin) | [| _prog; "--worker"; hash |] -> Solver.main (Git_unix.Store.Hash.of_hex hash) - | [| _prog; "test"; |] -> - Solver.test (Git_unix.Store.Hash.of_hex "ac01ad6037c0bdcca9f67fe49cd54475b585f9b2") + | [| _prog; "test" |] -> + Solver.test + (Git_unix.Store.Hash.of_hex "ac01ad6037c0bdcca9f67fe49cd54475b585f9b2") | args -> Fmt.failwith "Usage: ocaml-ci-solver (got %a)" Fmt.(array (quote string)) File "src/lib/epoch.mli", line 1, characters 0-0: diff --git a/_build/default/src/lib/epoch.mli b/_build/default/src/lib/.formatted/epoch.mli index e31dd99..fa7fab7 100644 --- a/_build/default/src/lib/epoch.mli +++ b/_build/default/src/lib/.formatted/epoch.mli @@ -1,6 +1,5 @@ type t val v : Config.t -> t - val digest : t -> string val pp : t Fmt.t File "src/lib/live.ml", line 1, characters 0-0: diff --git a/_build/default/src/lib/live.ml b/_build/default/src/lib/.formatted/live.ml index 9ae3877..1901f07 100644 --- a/_build/default/src/lib/live.ml +++ b/_build/default/src/lib/.formatted/live.ml @@ -4,9 +4,7 @@ let set_current ~ssh name generation = let open Current.Syntax in Current.component "Set current folder" |> let> generation in - let new_generation_folder = - Storage.Base.generation_folder generation - in + let new_generation_folder = Storage.Base.generation_folder generation in let storage_folder = Fpath.(of_string (Ssh.storage_folder ssh) |> Result.get_ok) in @@ -18,9 +16,7 @@ let set_live ~ssh name generation = let open Current.Syntax in Current.component "Set live folder" |> let> generation in - let new_generation_folder = - Storage.Base.generation_folder generation - in + let new_generation_folder = Storage.Base.generation_folder generation in let storage_folder = Fpath.(of_string (Ssh.storage_folder ssh) |> Result.get_ok) in @@ -30,6 +26,4 @@ let set_live ~ssh name generation = let set_to ~ssh name generation = Current.all - [ - set_current ~ssh name generation; set_live ~ssh name generation; - ] + [ set_current ~ssh name generation; set_live ~ssh name generation ] File "src/lib/compile.mli", line 1, characters 0-0: diff --git a/_build/default/src/lib/compile.mli b/_build/default/src/lib/.formatted/compile.mli index a538cdf..4dc2238 100644 --- a/_build/default/src/lib/compile.mli +++ b/_build/default/src/lib/.formatted/compile.mli @@ -5,13 +5,19 @@ the compilation, link and html generation steps, outputting the results in the compile/ and html/ folders. *) -type hashes = { compile_hash : string option; linked_hash : string option; html_hash : string option } +type hashes = { + compile_hash : string option; + linked_hash : string option; + html_hash : string option; +} type t (** A compiled package *) -type jobty = | CompileAndLink | CompileOnly | LinkOnly -(** The type of compilation job *) +type jobty = + | CompileAndLink + | CompileOnly + | LinkOnly (** The type of compilation job *) val hashes : t -> hashes (** Hash of the compiled artifacts *) File "src/lib/live.mli", line 1, characters 0-0: diff --git a/_build/default/src/lib/live.mli b/_build/default/src/lib/.formatted/live.mli index 30fd180..d4e67fc 100644 --- a/_build/default/src/lib/live.mli +++ b/_build/default/src/lib/.formatted/live.mli @@ -1,5 +1 @@ -val set_to : - ssh:Config.Ssh.t -> - string -> - Epoch.t Current.t -> - unit Current.t +val set_to : ssh:Config.Ssh.t -> string -> Epoch.t Current.t -> unit Current.t File "src/lib/record.ml", line 1, characters 0-0: diff --git a/_build/default/src/lib/record.ml b/_build/default/src/lib/.formatted/record.ml index 7b71be8..c72f873 100644 --- a/_build/default/src/lib/record.ml +++ b/_build/default/src/lib/.formatted/record.ml @@ -1,4 +1,3 @@ - let v config = let open Lwt.Syntax in let generation = Epoch.v config in @@ -7,9 +6,7 @@ let v config = let epoch_html = Epoch.digest generation in let result = - Index.record_new_pipeline - ~odoc_commit ~epoch_html - ~epoch_linked + Index.record_new_pipeline ~odoc_commit ~epoch_html ~epoch_linked in match result with | Ok pipeline_id -> Lwt.return_ok (pipeline_id |> Int64.to_int) File "src/lib/prep.mli", line 1, characters 0-0: diff --git a/_build/default/src/lib/prep.mli b/_build/default/src/lib/.formatted/prep.mli index ce07fe0..5e25d50 100644 --- a/_build/default/src/lib/prep.mli +++ b/_build/default/src/lib/.formatted/prep.mli @@ -1,8 +1,7 @@ type t (** The type for a prepped package (build objects in a universe/package folder) *) -module OpamFiles :sig - +module OpamFiles : sig type t = No_context module Key : sig @@ -10,15 +9,17 @@ module OpamFiles :sig end module Value : sig - type t = ((bool * string), [`Msg of string]) result OpamPackage.Map.t + type t = (bool * string, [ `Msg of string ]) result OpamPackage.Map.t [@@deriving yojson] end end module OpamFilesCache : sig - val get : ?schedule:Current_cache.Schedule.t -> + val get : + ?schedule:Current_cache.Schedule.t -> OpamFiles.t -> - OpamFiles.Key.t -> OpamFiles.Value.t Current.Primitive.t + OpamFiles.Key.t -> + OpamFiles.Value.t Current.Primitive.t end val hash : t -> string ocamlformat: ignoring "src/lib/misc.ml" (misplaced documentation comments - warning 50) File "src/lib/misc.ml", line 64, characters 0-31: 64 | (** Select base image to use *) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Warning 50 [unexpected-docstring]: ambiguous documentation comment Hint: (Warning 50) This file contains a documentation comment (** ... *) that the OCaml compiler does not know how to attach to the AST. OCamlformat does not support these cases. You can find more information at: https://github.com/ocaml-ppx/ocamlformat#overview. If you'd like to disable this check and let ocamlformat make a choice (though it might not be consistent with the ocaml compilers and odoc), you can set the --no-comment-check option. -> required by _build/default/src/lib/.formatted/misc.ml -> required by alias src/lib/.formatted/fmt -> required by alias src/lib/fmt File "src/lib/storage.mli", line 1, characters 0-0: diff --git a/_build/default/src/lib/storage.mli b/_build/default/src/lib/.formatted/storage.mli index 5230ae1..b86d55d 100644 --- a/_build/default/src/lib/storage.mli +++ b/_build/default/src/lib/.formatted/storage.mli @@ -8,7 +8,12 @@ type repository = val folder : repository -> Package.t -> Fpath.t module Base : sig - type repository = HtmlRaw of Epoch.t | Linked of Epoch.t | Compile of Epoch.t | Prep | Prep0 + type repository = + | HtmlRaw of Epoch.t + | Linked of Epoch.t + | Compile of Epoch.t + | Prep + | Prep0 val folder : repository -> Fpath.t val generation_folder : Epoch.t -> Fpath.t File "src/lib/epoch.ml", line 1, characters 0-0: diff --git a/_build/default/src/lib/epoch.ml b/_build/default/src/lib/.formatted/epoch.ml index d10e48a..1b22f57 100644 --- a/_build/default/src/lib/epoch.ml +++ b/_build/default/src/lib/.formatted/epoch.ml @@ -1,15 +1,15 @@ -type t = { config : Config.t; } +type t = { config : Config.t } let version = "v2" -let v config = { config; } +let v config = { config } let digest t = let key = - Fmt.str "%s:%s:%s" version (Config.odoc t.config) (Config.sherlodoc t.config) + Fmt.str "%s:%s:%s" version (Config.odoc t.config) + (Config.sherlodoc t.config) in key |> Digest.string |> Digest.to_hex let pp f t = - Fmt.pf f - "docs-ci: %s\nodoc: %s\nsherlodoc: %s" - version (Config.odoc t.config) (Config.sherlodoc t.config) + Fmt.pf f "docs-ci: %s\nodoc: %s\nsherlodoc: %s" version (Config.odoc t.config) + (Config.sherlodoc t.config) File "src/lib/config.mli", line 1, characters 0-0: diff --git a/_build/default/src/lib/config.mli b/_build/default/src/lib/.formatted/config.mli index 77b0449..5d05c31 100644 --- a/_build/default/src/lib/config.mli +++ b/_build/default/src/lib/.formatted/config.mli @@ -48,4 +48,4 @@ val take_n_last_versions : t -> int option (** Number of versions to take (None for all) *) val cache_threshold : t -> int -(** Number of versions to keep in the cache *) \ No newline at end of file +(** Number of versions to keep in the cache *) File "src/solver/solver.ml", line 1, characters 0-0: diff --git a/_build/default/src/solver/solver.ml b/_build/default/src/solver/.formatted/solver.ml index cd90716..956eb9f 100644 --- a/_build/default/src/solver/solver.ml +++ b/_build/default/src/solver/.formatted/solver.ml @@ -14,53 +14,59 @@ let env (vars : Worker.Vars.t) = let get_names = OpamFormula.fold_left (fun a (name, _) -> name :: a) [] let universes ~packages (resolutions : OpamPackage.t list) = - let aux root = - let name, version = (OpamPackage.name root, OpamPackage.version root) in - let opamfile : OpamFile.OPAM.t = - packages - |> OpamPackage.Name.Map.find name - |> OpamPackage.Version.Map.find version - in - let deps = - opamfile - |> OpamFile.OPAM.depends - |> OpamFilter.partial_filter_formula - (OpamFilter.deps_var_env ~build:true ~post:false ~test:false - ~doc:false ~dev_setup:false ~dev:false) - |> get_names - |> OpamPackage.Name.Set.of_list - in - let depopts = - opamfile - |> OpamFile.OPAM.depopts - |> OpamFilter.partial_filter_formula - (OpamFilter.deps_var_env ~build:true ~post:false ~test:false - ~doc:false ~dev_setup:false ~dev:false) - |> get_names - |> OpamPackage.Name.Set.of_list - in - let all_deps = OpamPackage.Name.Set.union deps depopts in - let deps = - resolutions - |> List.filter (fun res -> - let name = OpamPackage.name res in - OpamPackage.Name.Set.mem name all_deps) - in - let result = - OpamPackage.Set.of_list deps - in - result + let name, version = (OpamPackage.name root, OpamPackage.version root) in + let opamfile : OpamFile.OPAM.t = + packages + |> OpamPackage.Name.Map.find name + |> OpamPackage.Version.Map.find version + in + let deps = + opamfile + |> OpamFile.OPAM.depends + |> OpamFilter.partial_filter_formula + (OpamFilter.deps_var_env ~build:true ~post:false ~test:false + ~doc:false ~dev_setup:false ~dev:false) + |> get_names + |> OpamPackage.Name.Set.of_list + in + let depopts = + opamfile + |> OpamFile.OPAM.depopts + |> OpamFilter.partial_filter_formula + (OpamFilter.deps_var_env ~build:true ~post:false ~test:false + ~doc:false ~dev_setup:false ~dev:false) + |> get_names + |> OpamPackage.Name.Set.of_list + in + let all_deps = OpamPackage.Name.Set.union deps depopts in + let deps = + resolutions + |> List.filter (fun res -> + let name = OpamPackage.name res in + OpamPackage.Name.Set.mem name all_deps) + in + let result = OpamPackage.Set.of_list deps in + result + in + let simple_deps = + List.fold_left + (fun acc pkg -> OpamPackage.Map.add pkg (aux pkg) acc) + OpamPackage.Map.empty resolutions in - let simple_deps = List.fold_left (fun acc pkg -> OpamPackage.Map.add pkg (aux pkg) acc) OpamPackage.Map.empty resolutions in - + let rec closure pkgs = - let deps = List.map (fun pkg -> OpamPackage.Map.find pkg simple_deps) (OpamPackage.Set.to_list pkgs) in + let deps = + List.map + (fun pkg -> OpamPackage.Map.find pkg simple_deps) + (OpamPackage.Set.to_list pkgs) + in let all = List.fold_left OpamPackage.Set.union OpamPackage.Set.empty deps in let new_deps = OpamPackage.Set.diff all pkgs in - if OpamPackage.Set.is_empty new_deps then pkgs else closure (OpamPackage.Set.union pkgs all) + if OpamPackage.Set.is_empty new_deps then pkgs + else closure (OpamPackage.Set.union pkgs all) in - + List.rev_map (fun pkg -> let name, version = (OpamPackage.name pkg, OpamPackage.version pkg) in @@ -70,16 +76,23 @@ let universes ~packages (resolutions : OpamPackage.t list) = |> OpamPackage.Version.Map.find version in let str = OpamFile.OPAM.write_to_string opamfile in - (pkg, str, closure (OpamPackage.Map.find pkg simple_deps) |> OpamPackage.Set.elements)) - resolutions + ( pkg, + str, + closure (OpamPackage.Map.find pkg simple_deps) + |> OpamPackage.Set.elements )) + resolutions -type solve_result = Worker.solve_result = - { compile_universes : (string * string * string list) list; - link_universes: (string * string * string list) list; } [@@deriving yojson] +type solve_result = Worker.solve_result = { + compile_universes : (string * string * string list) list; + link_universes : (string * string * string list) list; +} +[@@deriving yojson] let solve ~packages ~constraints ~root_pkgs (vars : Worker.Vars.t) = let extended = Git_context.extend_packages packages in - let context = Git_context.create () ~packages:extended ~env:(env vars) ~constraints in + let context = + Git_context.create () ~packages:extended ~env:(env vars) ~constraints + in let t0 = Unix.gettimeofday () in let r = Solver.solve context root_pkgs in let t1 = Unix.gettimeofday () in @@ -99,12 +112,12 @@ let solve ~packages ~constraints ~root_pkgs (vars : Worker.Vars.t) = univs in Ok - { compile_universes = map_universes compile_universes; - link_universes = map_universes link_universes } + { + compile_universes = map_universes compile_universes; + link_universes = map_universes link_universes; + } | Error diagnostics -> Error (Solver.diagnostics diagnostics) - - let test commit = Format.eprintf "Running test\n%!"; let packages = @@ -112,27 +125,38 @@ let test commit = ( Opam_repository.open_store () >>= fun store -> Git_context.read_packages store commit ) in - let root_pkgs = List.map OpamPackage.Name.of_string ["ocaml";"ocaml-base-compiler";"vpnkit"] in - let constraints = - List.map (fun (name, rel, version) -> - ( OpamPackage.Name.of_string name, - (rel, OpamPackage.Version.of_string version) )) [("ocaml-base-compiler",`Geq, "4.08.0"); ("ocaml", `Leq, "5.2.0"); ("vpnkit", `Eq, "0.2.0")] - |> OpamPackage.Name.Map.of_list - in - let platform = { - Worker.Vars.arch = "x86_64"; - os="linux"; - os_distribution="linux"; - os_family="ubuntu"; - os_version="20.04"; - } in - Format.eprintf "Calling solve\n%!"; - match solve ~packages ~constraints ~root_pkgs platform with - | Ok packages -> - - Printf.printf "%s\n" (solve_result_to_yojson packages |> Yojson.Safe.to_string) - | Error msg -> Printf.printf "%s\n" msg - + let root_pkgs = + List.map OpamPackage.Name.of_string + [ "ocaml"; "ocaml-base-compiler"; "vpnkit" ] + in + let constraints = + List.map + (fun (name, rel, version) -> + ( OpamPackage.Name.of_string name, + (rel, OpamPackage.Version.of_string version) )) + [ + ("ocaml-base-compiler", `Geq, "4.08.0"); + ("ocaml", `Leq, "5.2.0"); + ("vpnkit", `Eq, "0.2.0"); + ] + |> OpamPackage.Name.Map.of_list + in + let platform = + { + Worker.Vars.arch = "x86_64"; + os = "linux"; + os_distribution = "linux"; + os_family = "ubuntu"; + os_version = "20.04"; + } + in + Format.eprintf "Calling solve\n%!"; + match solve ~packages ~constraints ~root_pkgs platform with + | Ok packages -> + Printf.printf "%s\n" + (solve_result_to_yojson packages |> Yojson.Safe.to_string) + | Error msg -> Printf.printf "%s\n" msg + let main commit = let packages = Lwt_main.run File "src/lib/storage.ml", line 1, characters 0-0: diff --git a/_build/default/src/lib/storage.ml b/_build/default/src/lib/.formatted/storage.ml index 55903c6..9c42275 100644 --- a/_build/default/src/lib/storage.ml +++ b/_build/default/src/lib/.formatted/storage.ml @@ -1,16 +1,18 @@ module Base = struct - type repository = HtmlRaw of Epoch.t | Linked of Epoch.t | Compile of Epoch.t | Prep | Prep0 + type repository = + | HtmlRaw of Epoch.t + | Linked of Epoch.t + | Compile of Epoch.t + | Prep + | Prep0 let generation_folder generation = Fpath.(v ("epoch-" ^ Epoch.digest generation)) let folder = function - | HtmlRaw generation -> - Fpath.(generation_folder generation / "html-raw") - | Linked generation -> - Fpath.(generation_folder generation / "linked") - | Compile generation -> - Fpath.(generation_folder generation / "compile") + | HtmlRaw generation -> Fpath.(generation_folder generation / "html-raw") + | Linked generation -> Fpath.(generation_folder generation / "linked") + | Compile generation -> Fpath.(generation_folder generation / "compile") | Prep -> Fpath.v "prep" | Prep0 -> Fpath.v "prep0" end @@ -50,17 +52,16 @@ let folder repository package = Base.folder (to_base_repo repository) // base_folder ~blessed ~prep:(repository = Prep) package) -let cache repository package = - folder repository package - +let cache repository package = folder repository package + let split packages = let rec take n l = - match n,l with - | 0, _ -> [], l - | n, x::xs -> - let taken, rest = take (n-1) xs in - x :: taken, rest - | _, [] -> [], [] + match (n, l) with + | 0, _ -> ([], l) + | n, x :: xs -> + let taken, rest = take (n - 1) xs in + (x :: taken, rest) + | _, [] -> ([], []) in let rec run remaining = match take 10 remaining with @@ -91,13 +92,15 @@ module Tar = struct match extra_files with | [] -> Fmt.str - "HASH=$((sha256sum $1/content.tar 2>/dev/null | cut -d \" \" -f 1) || echo -n \ - 'empty'); rm -f $1/content.tar; printf \"%s:$2:$HASH\\n\";" + "HASH=$((sha256sum $1/content.tar 2>/dev/null | cut -d \" \" -f 1) \ + || echo -n 'empty'); rm -f $1/content.tar; printf \ + \"%s:$2:$HASH\\n\";" prefix | extra_files -> Fmt.str "HASH=$((sha256sum $1/content.tar %s | sort | sha256sum | cut -d \" \ - \" -f 1) || echo -n 'empty'); rm $1/content.tar; printf \"%s:$2:$HASH\\n\";" + \" -f 1) || echo -n 'empty'); rm $1/content.tar; printf \ + \"%s:$2:$HASH\\n\";" (List.map (fun f -> "\"$1/" ^ f ^ "\"") extra_files |> String.concat " ") prefix File "src/lib/voodoo.ml", line 1, characters 0-0: diff --git a/_build/default/src/lib/voodoo.ml b/_build/default/src/lib/.formatted/voodoo.ml index fead1c5..719d27c 100644 --- a/_build/default/src/lib/voodoo.ml +++ b/_build/default/src/lib/.formatted/voodoo.ml @@ -12,13 +12,18 @@ module Prep = struct base |> Spec.add [ - run ~network ~cache "%s" @@ Misc.Cmd.list [ - "sudo apt-get update && sudo apt-get install -yy m4 pkg-config"; - Fmt.str "opam pin -ny opamh %s" - "https://github.com/jonludlam/opamh.git#d54dbe7991af6020c36f89c300d8109664541963 && opam install -y opamh"; - "cp $(opam config var bin)/opamh /home/opam"; - "rm -rf $(opam config var prefix)" - ]] + run ~network ~cache "%s" + @@ Misc.Cmd.list + [ + "sudo apt-get update && sudo apt-get install -yy m4 \ + pkg-config"; + Fmt.str "opam pin -ny opamh %s" + "https://github.com/jonludlam/opamh.git#d54dbe7991af6020c36f89c300d8109664541963 \ + && opam install -y opamh"; + "cp $(opam config var bin)/opamh /home/opam"; + "rm -rf $(opam config var prefix)"; + ]; + ] end module Odoc = struct @@ -27,35 +32,44 @@ module Odoc = struct base |> Spec.add [ - run ~network ~cache "%s" @@ Misc.Cmd.list [ - "sudo apt-get update && sudo apt-get install -yy m4"; - Fmt.str "opam pin -ny odoc-parser.dev %s && opam depext -iy odoc-parser" - (Config.odoc config); - Fmt.str "opam pin -ny odoc.dev %s && opam depext -iy odoc" - (Config.odoc config); - "cp $(opam config var bin)/odoc /home/opam"; - "rm -rf $(opam config var prefix)" - ]] + run ~network ~cache "%s" + @@ Misc.Cmd.list + [ + "sudo apt-get update && sudo apt-get install -yy m4"; + Fmt.str + "opam pin -ny odoc-parser.dev %s && opam depext -iy \ + odoc-parser" + (Config.odoc config); + Fmt.str "opam pin -ny odoc.dev %s && opam depext -iy odoc" + (Config.odoc config); + "cp $(opam config var bin)/odoc /home/opam"; + "rm -rf $(opam config var prefix)"; + ]; + ] end module OdocDriver = struct - let spec ~base ~odoc_pin ~sherlodoc_pin = let open Obuilder_spec in base |> Spec.add [ - run ~network ~cache "%s" @@ - Misc.Cmd.list - ["sudo apt-get update"; - Fmt.str "opam pin -ny sherlodoc.dev %s" - sherlodoc_pin; - Fmt.str "opam pin -ny odoc-parser.dev %s && opam pin -ny odoc-md.dev %s && opam pin -ny odoc.dev %s && opam pin -ny odoc-driver.dev %s" - odoc_pin odoc_pin odoc_pin odoc_pin; - "opam install -y odoc-md"; - "opam install -y odoc-driver sherlodoc"; - "cp $(opam config var bin)/odoc_driver_voodoo $(opam config var bin)/sherlodoc $(opam config var bin)/odoc-md /home/opam"; - "rm -rf $(opam config var prefix)"] + run ~network ~cache "%s" + @@ Misc.Cmd.list + [ + "sudo apt-get update"; + Fmt.str "opam pin -ny sherlodoc.dev %s" sherlodoc_pin; + Fmt.str + "opam pin -ny odoc-parser.dev %s && opam pin -ny \ + odoc-md.dev %s && opam pin -ny odoc.dev %s && opam pin \ + -ny odoc-driver.dev %s" + odoc_pin odoc_pin odoc_pin odoc_pin; + "opam install -y odoc-md"; + "opam install -y odoc-driver sherlodoc"; + "cp $(opam config var bin)/odoc_driver_voodoo $(opam config \ + var bin)/sherlodoc $(opam config var bin)/odoc-md \ + /home/opam"; + "rm -rf $(opam config var prefix)"; + ]; ] - end File "src/lib/track.ml", line 1, characters 0-0: diff --git a/_build/default/src/lib/track.ml b/_build/default/src/lib/.formatted/track.ml index eeb24ed..d7de969 100644 --- a/_build/default/src/lib/track.ml +++ b/_build/default/src/lib/.formatted/track.ml @@ -81,7 +81,8 @@ module Track = struct let filter name = match filter with [] -> true | lst -> List.mem (Fpath.basename name) lst in - Log.info (fun f -> f "Tracking packages in %a" Fpath.pp (Git.Commit.repo repo)); + Log.info (fun f -> + f "Tracking packages in %a" Fpath.pp (Git.Commit.repo repo)); let* () = Current.Job.start ~level:Harmless job in Git.with_checkout ~job repo @@ fun dir -> let result = File "src/solver/service.ml", line 1, characters 0-0: diff --git a/_build/default/src/solver/service.ml b/_build/default/src/solver/.formatted/service.ml index 82386c9..d84705a 100644 --- a/_build/default/src/solver/service.ml +++ b/_build/default/src/solver/.formatted/service.ml @@ -92,15 +92,20 @@ end = struct | _ -> Fmt.failwith "BUG: bad output: %s" results) let handle ~log request t = - let { Worker.Solve_request.opam_repository_commit; platforms; pkgs; constraints } = + let { + Worker.Solve_request.opam_repository_commit; + platforms; + pkgs; + constraints; + } = request in - Log.info log "Solving for %a, constraints %a using opam_repository_commit %s" + Log.info log + "Solving for %a, constraints %a using opam_repository_commit %s" Fmt.(list ~sep:comma string) pkgs Fmt.(list ~sep:comma Worker.Solve_request.pp_constraint) - constraints - opam_repository_commit; + constraints opam_repository_commit; let opam_repository_commit = Store.Hash.of_hex opam_repository_commit in platforms |> Lwt_list.map_p (fun p -> @@ -110,7 +115,9 @@ end = struct | Error _ as e -> Lwt.return (id, e) | Ok packages -> let repo_packages = - List.map (fun (pkg, _opam, _) -> OpamPackage.of_string pkg) packages.link_universes + List.map + (fun (pkg, _opam, _) -> OpamPackage.of_string pkg) + packages.link_universes in Opam_repository.oldest_commit_with repo_packages ~from:opam_repository_commit ~log File "src/lib/track.mli", line 1, characters 0-0: diff --git a/_build/default/src/lib/track.mli b/_build/default/src/lib/.formatted/track.mli index 9681d96..50774e3 100644 --- a/_build/default/src/lib/track.mli +++ b/_build/default/src/lib/.formatted/track.mli @@ -2,6 +2,7 @@ type t [@@deriving yojson] val digest : t -> string val pkg : t -> OpamPackage.t + val v : limit:int option -> filter:string list -> File "src/solver/git_context.ml", line 1, characters 0-0: diff --git a/_build/default/src/solver/git_context.ml b/_build/default/src/solver/.formatted/git_context.ml index e3c302b..e5dd648 100644 --- a/_build/default/src/solver/git_context.ml +++ b/_build/default/src/solver/.formatted/git_context.ml @@ -28,8 +28,8 @@ let filter_deps t pkg f = let test = OpamPackage.Name.Set.mem (OpamPackage.name pkg) t.test in f |> OpamFilter.partial_filter_formula (env t pkg) - |> OpamFilter.filter_deps ~build:true ~post:true ~test ~doc:false ~dev ~dev_setup:false - ~default:false + |> OpamFilter.filter_deps ~build:true ~post:true ~test ~doc:false ~dev + ~dev_setup:false ~default:false let candidates t name = match OpamPackage.Name.Map.find_opt name t.pins with @@ -89,22 +89,34 @@ let read_package store pkg hash = ) let extend_packages packages = - OpamPackage.Name.Map.map (fun versions -> - OpamPackage.Version.Map.map (fun opam -> - let extensions = OpamFile.OPAM.extensions opam in - let pp = OpamFormat.V.(package_formula `Conj (filtered_constraints ext_version)) in - (try - let extra_doc_deps = OpamStd.String.Map.find "x-extra-doc-deps" extensions in - let raw = OpamPp.parse pp ~pos:OpamTypesBase.pos_null extra_doc_deps in - let deps = OpamFile.OPAM.depends opam in - let x = (OpamFormula.ands_to_list raw) in - let y = (OpamFormula.ands_to_list deps) in - let deps = x @ y |> OpamFormula.ands in - let opam = OpamFile.OPAM.with_depends deps opam in - Format.eprintf "Extended dependencies for %s:\n%!" (OpamFile.OPAM.bug_reports opam |> String.concat ", "); - Format.eprintf "%s\n%!" (OpamFile.OPAM.write_to_string opam); - opam - with Not_found -> opam)) versions) packages + OpamPackage.Name.Map.map + (fun versions -> + OpamPackage.Version.Map.map + (fun opam -> + let extensions = OpamFile.OPAM.extensions opam in + let pp = + OpamFormat.V.( + package_formula `Conj (filtered_constraints ext_version)) + in + try + let extra_doc_deps = + OpamStd.String.Map.find "x-extra-doc-deps" extensions + in + let raw = + OpamPp.parse pp ~pos:OpamTypesBase.pos_null extra_doc_deps + in + let deps = OpamFile.OPAM.depends opam in + let x = OpamFormula.ands_to_list raw in + let y = OpamFormula.ands_to_list deps in + let deps = x @ y |> OpamFormula.ands in + let opam = OpamFile.OPAM.with_depends deps opam in + Format.eprintf "Extended dependencies for %s:\n%!" + (OpamFile.OPAM.bug_reports opam |> String.concat ", "); + Format.eprintf "%s\n%!" (OpamFile.OPAM.write_to_string opam); + opam + with Not_found -> opam) + versions) + packages (* Get a map of the versions inside [entry] (an entry under "packages") *) let read_versions store (entry : Store.Value.Tree.entry) = File "src/lib/index.ml", line 1, characters 0-0: diff --git a/_build/default/src/lib/index.ml b/_build/default/src/lib/.formatted/index.ml index e4e6043..7d4ba76 100644 --- a/_build/default/src/lib/index.ml +++ b/_build/default/src/lib/.formatted/index.ml @@ -126,8 +126,8 @@ let db = pipeline_id = ?" and get_pipeline_data = Sqlite3.prepare db - "SELECT epoch_html, epoch_linked, odoc_commit FROM docs_ci_pipeline_index \ - WHERE id = ?" + "SELECT epoch_html, epoch_linked, odoc_commit FROM \ + docs_ci_pipeline_index WHERE id = ?" in { @@ -164,12 +164,7 @@ let record_new_pipeline ~odoc_commit ~epoch_html ~epoch_linked = let t = Lazy.force db in match Db.query_one t.record_pipeline - Sqlite3.Data. - [ - TEXT epoch_html; - TEXT epoch_linked; - TEXT odoc_commit; - ] + Sqlite3.Data.[ TEXT epoch_html; TEXT epoch_linked; TEXT odoc_commit ] with | Sqlite3.Data.[ INT pipeline_id ] -> Ok pipeline_id | _ -> Error "Failed to record pipeline." @@ -251,33 +246,15 @@ let get_pipeline_data pipeline_id = NULL; NULL; ] -> - ( epoch_html, - epoch_linked, - "" - ) - | Sqlite3.Data. - [ - TEXT epoch_html; - TEXT epoch_linked; - TEXT odoc_commit; - ] -> - ( epoch_html, - epoch_linked, - odoc_commit ) + (epoch_html, epoch_linked, "") + | Sqlite3.Data.[ TEXT epoch_html; TEXT epoch_linked; TEXT odoc_commit ] + -> + (epoch_html, epoch_linked, odoc_commit) | row -> Fmt.failwith "get_pipeline_data: invalid row %a" Db.dump_row row in match result with - | [ - ( epoch_html, - epoch_linked, - odoc_commit ); - ] -> - Some - { - epoch_html; - epoch_linked; - odoc_commit; - } + | [ (epoch_html, epoch_linked, odoc_commit) ] -> + Some { epoch_html; epoch_linked; odoc_commit } | _ -> None (* packages - (name, version) that are failing in the latest pipeline that are passing in the latest but one *) File "src/lib/package.mli", line 1, characters 0-0: diff --git a/_build/default/src/lib/package.mli b/_build/default/src/lib/.formatted/package.mli index 912665e..cbcb906 100644 --- a/_build/default/src/lib/package.mli +++ b/_build/default/src/lib/.formatted/package.mli @@ -22,6 +22,7 @@ and Package : sig end type t = Package.t + val to_yojson : t -> Yojson.Safe.t val of_yojson : Yojson.Safe.t -> t Ppx_deriving_yojson_runtime.error_or File "src/lib/package.ml", line 1, characters 0-0: diff --git a/_build/default/src/lib/package.ml b/_build/default/src/lib/.formatted/package.ml index 8462a4a..63d7d9a 100644 --- a/_build/default/src/lib/package.ml +++ b/_build/default/src/lib/.formatted/package.ml @@ -1,17 +1,17 @@ module Ocaml_version = struct include Ocaml_version - let to_yojson x = - `String (Ocaml_version.to_string x) + let to_yojson x = `String (Ocaml_version.to_string x) let of_yojson x = match x with - | `String s -> - (match Ocaml_version.of_string s with | Ok x -> Ok x | Error (`Msg s) -> Error s) - | _ -> Error ("Ocaml_version.of_yojson") + | `String s -> ( + match Ocaml_version.of_string s with + | Ok x -> Ok x + | Error (`Msg s) -> Error s) + | _ -> Error "Ocaml_version.of_yojson" end - module rec Universe : sig type t [@@deriving yojson] @@ -24,15 +24,18 @@ module rec Universe : sig val v : Ocaml_version.t -> Package.t list -> t val compare : t -> t -> int end = struct - type t = { ocaml_version : Ocaml_version.t; hash : string; deps : Package.t list; mutable extra_link_deps : Package.t list; } [@@deriving yojson] + type t = { + ocaml_version : Ocaml_version.t; + hash : string; + deps : Package.t list; + mutable extra_link_deps : Package.t list; + } + [@@deriving yojson] let hash t = t.hash let deps t = t.deps - let ocaml_version t = t.ocaml_version - let set_extra_link_deps t eld = t.extra_link_deps <- eld - let extra_link_deps t = t.extra_link_deps let v ocaml_version deps = @@ -65,7 +68,7 @@ and Package : sig val v : Ocaml_version.t -> OpamPackage.t -> t list -> string -> t val make : - ocaml_version : Ocaml_version.t -> + ocaml_version:Ocaml_version.t -> blacklist:string list -> commit:string -> root:OpamPackage.t -> @@ -73,7 +76,12 @@ and Package : sig (OpamPackage.t * OpamPackage.t list) list -> t end = struct - type t = { ocaml_version : Ocaml_version.t; opam : O.OpamPackage.t; universe : Universe.t; commit : string } + type t = { + ocaml_version : Ocaml_version.t; + opam : O.OpamPackage.t; + universe : Universe.t; + commit : string; + } [@@deriving yojson] let universe t = t.universe @@ -81,7 +89,9 @@ end = struct let commit t = t.commit let id t = OpamPackage.to_string t.opam ^ "-" ^ Universe.hash t.universe let digest = id - let v ocaml_version opam deps commit = { ocaml_version; opam; universe = Universe.v ocaml_version deps; commit } + + let v ocaml_version opam deps commit = + { ocaml_version; opam; universe = Universe.v ocaml_version deps; commit } let pp f { universe; opam; _ } = Fmt.pf f "%s; %a" (OpamPackage.to_string opam) Universe.pp universe @@ -121,47 +131,60 @@ end = struct pkg in let results = obtain root in - OpamPackage.Map.iter (fun opam_package pkg -> - let compile_deps = List.assoc opam_package compile_deps |> OpamPackage.Set.of_list in - let link_deps = List.assoc opam_package link_deps |> OpamPackage.Set.of_list in - let extras = OpamPackage.Set.diff link_deps compile_deps in - let extras = - if OpamPackage.Set.cardinal extras = 0 then [] - else OpamPackage.Set.(elements (add opam_package extras)) - in - let extras = List.map (fun opam_package -> obtain opam_package) extras in - Universe.set_extra_link_deps pkg.universe extras - ) !memo; + OpamPackage.Map.iter + (fun opam_package pkg -> + let compile_deps = + List.assoc opam_package compile_deps |> OpamPackage.Set.of_list + in + let link_deps = + List.assoc opam_package link_deps |> OpamPackage.Set.of_list + in + let extras = OpamPackage.Set.diff link_deps compile_deps in + let extras = + if OpamPackage.Set.cardinal extras = 0 then [] + else OpamPackage.Set.(elements (add opam_package extras)) + in + let extras = + List.map (fun opam_package -> obtain opam_package) extras + in + Universe.set_extra_link_deps pkg.universe extras) + !memo; results end include Package let topo_sort packages = - let graph = List.map (fun pkg -> - let universe = universe pkg in - let deps = Universe.deps universe in - (pkg, deps) - ) packages + let graph = + List.map + (fun pkg -> + let universe = universe pkg in + let deps = Universe.deps universe in + (pkg, deps)) + packages in let rec loop graph = match graph with | [] -> [] - | _ -> - let zero_deps,others = List.partition (fun (_,x) -> x = []) graph in - let zero_deps = List.map fst zero_deps in - let other = List.map (fun (x,y) -> (x, List.filter (fun dep -> not (List.mem dep zero_deps)) y)) others in - let sorted = List.sort compare zero_deps in - sorted :: loop other + | _ -> + let zero_deps, others = List.partition (fun (_, x) -> x = []) graph in + let zero_deps = List.map fst zero_deps in + let other = + List.map + (fun (x, y) -> + (x, List.filter (fun dep -> not (List.mem dep zero_deps)) y)) + others + in + let sorted = List.sort compare zero_deps in + sorted :: loop other in List.flatten (loop graph) - +let all_deps pkg = + (pkg :: (pkg |> universe |> Universe.deps)) + @ (pkg |> universe |> Universe.extra_link_deps) -let all_deps pkg = pkg :: (pkg |> universe |> Universe.deps) @ (pkg |> universe |> Universe.extra_link_deps) - -let ocaml_version pkg = - pkg |> universe |> Universe.ocaml_version +let ocaml_version pkg = pkg |> universe |> Universe.ocaml_version module PackageMap = Map.Make (Package) module PackageSet = Set.Make (Package) @@ -240,8 +263,8 @@ module Map = PackageMap module Set = PackageSet let important_packages : Set.t ref = ref Set.empty + let add_important_packages packages = important_packages := Set.union !important_packages packages -let should_cache package = - Set.mem package !important_packages +let should_cache package = Set.mem package !important_packages File "src/lib/config.ml", line 1, characters 0-0: diff --git a/_build/default/src/lib/config.ml b/_build/default/src/lib/.formatted/config.ml index c958614..abfa2bb 100644 --- a/_build/default/src/lib/config.ml +++ b/_build/default/src/lib/.formatted/config.ml @@ -1,11 +1,12 @@ open Cmdliner let random = -(* In_channel.with_open_bin "/dev/urandom" (fun - ch -> - let str = String.init 4 (fun _ -> In_channel.input_byte ch |> Option.get |> Char.chr) in - let str = Base64.encode_exn str in - str)*) "NOTRANDOM" + (* In_channel.with_open_bin "/dev/urandom" (fun + ch -> + let str = String.init 4 (fun _ -> In_channel.input_byte ch |> Option.get |> Char.chr) in + let str = Base64.encode_exn str in + str)*) + "NOTRANDOM" module Ssh = struct type t = { @@ -173,11 +174,10 @@ let take_n_last_versions = let cache_threshold = Arg.value - @@ Arg.opt Arg.(int) (10) + @@ Arg.opt Arg.(int) 10 @@ Arg.info ~doc:"Cache threshold" ~docv:"THRESHOLD" [ "cache-threshold" ] -let v cap_file jobs track_packages - take_n_last_versions ssh cache_threshold = +let v cap_file jobs track_packages take_n_last_versions ssh cache_threshold = let vat = Capnp_rpc_unix.client_only_vat () in let cap = Capnp_rpc_unix.Cap_file.load vat cap_file |> Result.get_ok in @@ -217,7 +217,6 @@ let odoc _ = "https://github.com/ocaml/odoc.git#90e679061f68c5e5ee5915e280f63d842f41f300" let sherlodoc _ = odoc () - let pool _ = "docs-pipeline" let jobs t = t.jobs let track_packages t = t.track_packages @@ -226,4 +225,4 @@ let ocluster_connection_do t = t.ocluster_connection_do let ocluster_connection_prep t = t.ocluster_connection_prep let ocluster_connection_gen t = t.ocluster_connection_gen let ssh t = t.ssh -let cache_threshold t = t.cache_threshold \ No newline at end of file +let cache_threshold t = t.cache_threshold File "src/lib/solver.ml", line 1, characters 0-0: diff --git a/_build/default/src/lib/solver.ml b/_build/default/src/lib/.formatted/solver.ml index db63a0c..a727899 100644 --- a/_build/default/src/lib/solver.ml +++ b/_build/default/src/lib/.formatted/solver.ml @@ -74,7 +74,10 @@ let perform_constrained_solve ~solver ~pool ~job ~(platform : Platform.t) ~opam (OpamPackage.of_string a, List.map OpamPackage.of_string b)) u in - Ok (solution x.packages.compile_universes, solution x.packages.link_universes, x.commit) + Ok + ( solution x.packages.compile_universes, + solution x.packages.link_universes, + x.commit ) | Ok _ -> Fmt.error_msg "??" | Error (`Msg msg) -> Fmt.error_msg "Error from solver: %s" msg) (fun exn -> @@ -92,8 +95,7 @@ let perform_solve ~solver ~pool ~job ~(platform : Platform.t) ~opam track = in let latest = Ocaml_version.Releases.latest |> Ocaml_version.to_string in perform_constrained_solve ~solver ~pool ~job ~platform ~opam - (("ocaml-base-compiler", `Leq, latest) - :: constraints) + (("ocaml-base-compiler", `Leq, latest) :: constraints) let solver_version = "v2" @@ -128,10 +130,12 @@ module Cache = struct match value with | Error _ -> Printf.fprintf oc "Failed" | Ok pkg -> - let all = Package.all_deps pkg in - List.iter (fun pkg -> - let opam = Package.opam pkg in - Printf.fprintf oc "%s\n" (OpamPackage.to_string opam)) all + let all = Package.all_deps pkg in + List.iter + (fun pkg -> + let opam = Package.opam pkg in + Printf.fprintf oc "%s\n" (OpamPackage.to_string opam)) + all in close_out oc @@ -149,14 +153,17 @@ type key = Track.t type t = { successes : Track.t list; failures : Track.t list } let keys t = t.successes + let get key = - Format.eprintf "Trying to 'get' %s\n%!" (Track.pkg key |> OpamPackage.to_string); - Cache.read key |> Option.get (* is in cache ? *) |> Result.get_ok + Format.eprintf "Trying to 'get' %s\n%!" + (Track.pkg key |> OpamPackage.to_string); + Cache.read key |> Option.get (* is in cache ? *) |> Result.get_ok let failures t = t.failures |> List.filter_map (fun k -> - try Some (Track.pkg k, Cache.read k |> Option.get |> Result.get_error) with _ -> None) + try Some (Track.pkg k, Cache.read k |> Option.get |> Result.get_error) + with _ -> None) (* is solved ? *) @@ -214,22 +221,39 @@ module Solver = struct let root = Track.pkg pkg in let result = match res with - | Ok (compile_packages, link_packages, commit) -> + | Ok (compile_packages, link_packages, commit) -> ( Current.Job.log job "Packages returned: "; - begin try - let root_deps = List.assoc root compile_packages in - Current.Job.log job "Got the root package %s" (OpamPackage.to_string root); - let ocaml_version = List.find + try + let root_deps = List.assoc root compile_packages in + Current.Job.log job "Got the root package %s" + (OpamPackage.to_string root); + let ocaml_version = + List.find (fun p -> - Current.Job.log job "Checking %s" (OpamPackage.to_string p); - List.mem (OpamPackage.name_to_string p) ["ocaml-base-compiler"; "ocaml"]) (root::root_deps) |> OpamPackage.version_to_string |> Ocaml_version.of_string_exn in - Ok (Package.make ~ocaml_version ~blacklist ~commit ~root compile_packages link_packages) - with Not_found -> - Current.Job.log job "Package %s does not require OCaml" (OpamPackage.to_string root); - let colon = Fmt.any ":" in - Current.Job.log job "Packages returned: %a" Fmt.(list (pair ~sep:colon string (list ~sep:sp string))) (List.map (fun (a, b) -> (OpamPackage.to_string a, List.map OpamPackage.to_string b)) compile_packages); - Error ("No OCaml dependency") - end + Current.Job.log job "Checking %s" + (OpamPackage.to_string p); + List.mem + (OpamPackage.name_to_string p) + [ "ocaml-base-compiler"; "ocaml" ]) + (root :: root_deps) + |> OpamPackage.version_to_string + |> Ocaml_version.of_string_exn + in + Ok + (Package.make ~ocaml_version ~blacklist ~commit ~root + compile_packages link_packages) + with Not_found -> + Current.Job.log job "Package %s does not require OCaml" + (OpamPackage.to_string root); + let colon = Fmt.any ":" in + Current.Job.log job "Packages returned: %a" + Fmt.(list (pair ~sep:colon string (list ~sep:sp string))) + (List.map + (fun (a, b) -> + ( OpamPackage.to_string a, + List.map OpamPackage.to_string b )) + compile_packages); + Error "No OCaml dependency") | Error (`Msg msg) -> Current.Job.log job "Solving failed for %s: %s" (OpamPackage.to_string root) File "src/ocaml_docs_ci.ml", line 1, characters 0-0: diff --git a/_build/default/src/ocaml_docs_ci.ml b/_build/default/src/.formatted/ocaml_docs_ci.ml index 8b03642..720e87f 100644 --- a/_build/default/src/ocaml_docs_ci.ml +++ b/_build/default/src/.formatted/ocaml_docs_ci.ml @@ -108,7 +108,7 @@ let main () current_config github_auth mode capnp_public_address run_capnp capnp_public_address capnp_listen_address >>= fun (_vat, rpc_engine_resolver) -> let repo_opam = - Git.clone ~schedule:hourly + Git.clone ~schedule:hourly "https://github.com/ocaml/opam-repository.git" in let monitor = Docs_ci_lib.Monitor.make () in File "src/lib/compile.ml", line 1, characters 0-0: diff --git a/_build/default/src/lib/compile.ml b/_build/default/src/lib/.formatted/compile.ml index 2396983..06b0492 100644 --- a/_build/default/src/lib/compile.ml +++ b/_build/default/src/lib/.formatted/compile.ml @@ -1,8 +1,11 @@ -type hashes = { compile_hash : string option; linked_hash : string option; html_hash : string option } +type hashes = { + compile_hash : string option; + linked_hash : string option; + html_hash : string option; +} [@@deriving yojson] type t = { package : Package.t; blessing : Package.Blessing.t; hashes : hashes } - type jobty = CompileAndLink | CompileOnly | LinkOnly [@@deriving yojson] let pp_jobty = @@ -17,56 +20,72 @@ let hashes t = t.hashes let blessing t = t.blessing let package t = t.package -let spec_success ~ssh ~base ~odoc_driver_base ~odoc_pin ~sherlodoc_pin ~config ~deps ~blessing ~generation ~jobty prep = +let spec_success ~ssh ~base ~odoc_driver_base ~odoc_pin ~sherlodoc_pin ~config + ~deps ~blessing ~generation ~jobty prep = let open Obuilder_spec in let package = Prep.package prep in let prep_folder = Storage.folder Prep package in let prep0_folder = Storage.folder Prep0 package in - let compile_folder = Storage.folder (Compile (generation, blessing)) package in + let compile_folder = + Storage.folder (Compile (generation, blessing)) package + in let linked_folder = Storage.folder (Linked (generation, blessing)) package in let raw_folder = Storage.folder (HtmlRaw (generation, blessing)) package in let opam = package |> Package.opam in let name = opam |> OpamPackage.name_to_string in let tools = Voodoo.Odoc.spec ~base config |> Spec.finish in (* let compile_caches = - deps |> List.map (fun - {blessing; package; _} -> - let dir = Storage.folder (Compile blessing) package in - let fpath_strs = Fpath.segs dir in - let cache_name = String.concat "--" fpath_strs in - let dir = Fpath.(v "/home/opam/.cache/" // dir) in - Obuilder_spec.Cache.v cache_name ~target:(Fpath.to_string dir) - ) in *) + deps |> List.map (fun + {blessing; package; _} -> + let dir = Storage.folder (Compile blessing) package in + let fpath_strs = Fpath.segs dir in + let cache_name = String.concat "--" fpath_strs in + let dir = Fpath.(v "/home/opam/.cache/" // dir) in + Obuilder_spec.Cache.v cache_name ~target:(Fpath.to_string dir) + ) in *) let common = - (List.map (fun { blessing; package; _ } -> - let dir = Storage.folder (Compile (generation, blessing)) package in - Fmt.str "~/docs/docs-ci-scripts/download_compiled.sh %s %s %s %s %b" (Config.Ssh.host ssh) (Config.Ssh.storage_folder ssh) (Fpath.to_string dir) (Package.digest package ^ "-" ^ Epoch.digest generation) (Package.should_cache package) - ) deps) @ - [Fmt.str "~/docs/docs-ci-scripts/get_prep_for_compile.sh %s %s %s %s" (Config.Ssh.host ssh) (Config.Ssh.storage_folder ssh) (Fpath.to_string prep0_folder) (Fpath.to_string prep_folder); + List.map + (fun { blessing; package; _ } -> + let dir = Storage.folder (Compile (generation, blessing)) package in + Fmt.str "~/docs/docs-ci-scripts/download_compiled.sh %s %s %s %s %b" + (Config.Ssh.host ssh) + (Config.Ssh.storage_folder ssh) + (Fpath.to_string dir) + (Package.digest package ^ "-" ^ Epoch.digest generation) + (Package.should_cache package)) + deps + @ [ + Fmt.str "~/docs/docs-ci-scripts/get_prep_for_compile.sh %s %s %s %s" + (Config.Ssh.host ssh) + (Config.Ssh.storage_folder ssh) + (Fpath.to_string prep0_folder) + (Fpath.to_string prep_folder); "export OCAMLRUNPAM=b"; - ] in + ] + in let post_compile = [ - Misc.tar_cmd compile_folder; - Fmt.str "time rsync -aR ./%s %s:%s/." - (Fpath.to_string compile_folder) - (Config.Ssh.host ssh) - (Config.Ssh.storage_folder ssh); - Fmt.str "set '%s'; %s" - (Fpath.to_string compile_folder) - (Storage.Tar.hash_command ~prefix:"COMPILE" ())] + Misc.tar_cmd compile_folder; + Fmt.str "time rsync -aR ./%s %s:%s/." + (Fpath.to_string compile_folder) + (Config.Ssh.host ssh) + (Config.Ssh.storage_folder ssh); + Fmt.str "set '%s'; %s" + (Fpath.to_string compile_folder) + (Storage.Tar.hash_command ~prefix:"COMPILE" ()); + ] in let post_link_and_html = [ - Fmt.str "mkdir -p linked && mkdir -p %a && mv linked %a/" - Fpath.pp + Fmt.str "mkdir -p linked && mkdir -p %a && mv linked %a/" Fpath.pp (Storage.Base.generation_folder generation) Fpath.pp (Storage.Base.generation_folder generation); Fmt.str "mkdir -p %a" Fpath.pp linked_folder; Misc.tar_cmd linked_folder; Fmt.str "echo raw_folder: %s" (Fpath.to_string raw_folder); - Fmt.str "~/docs/docs-ci-scripts/gen_status_json.sh %s" (Fpath.to_string raw_folder); + Fmt.str "~/docs/docs-ci-scripts/gen_status_json.sh %s" + (Fpath.to_string raw_folder); Fmt.str "time rsync -aR ./%s %s:%s/." Fpath.(to_string (parent linked_folder)) (Config.Ssh.host ssh) @@ -86,51 +105,88 @@ let spec_success ~ssh ~base ~odoc_driver_base ~odoc_pin ~sherlodoc_pin ~config ~ Fmt.str "set '%s' raw; %s" (Fpath.to_string raw_folder) (Storage.hash_command ~prefix:"RAW"); - "rm -rf /home/opam/docs/*" - ] + "rm -rf /home/opam/docs/*"; + ] in let compile_caches = - [Obuilder_spec.Cache.v "compile-cache" ~target:(Fmt.str "/home/opam/.cache/%a/compile" Fpath.pp (Storage.Base.generation_folder generation))] in - let odoc_driver = Voodoo.OdocDriver.spec ~base:odoc_driver_base ~odoc_pin ~sherlodoc_pin |> Spec.finish in + [ + Obuilder_spec.Cache.v "compile-cache" + ~target: + (Fmt.str "/home/opam/.cache/%a/compile" Fpath.pp + (Storage.Base.generation_folder generation)); + ] + in + let odoc_driver = + Voodoo.OdocDriver.spec ~base:odoc_driver_base ~odoc_pin ~sherlodoc_pin + |> Spec.finish + in base |> Spec.children ~name:"tools" tools |> Spec.children ~name:"odoc_driver" odoc_driver |> Spec.add ([ - workdir "/home/opam/docs/"; - run "sudo chown opam:opam . "; - (* Import odoc and voodoo-do *) - copy ~from:(`Build "tools") - [ "/home/opam/odoc";] - ~dst:"/home/opam/"; - copy ~from:(`Build "odoc_driver") - [ "/home/opam/odoc_driver_voodoo"; "/home/opam/sherlodoc"; "/home/opam/odoc-md"; ] - ~dst:"/home/opam/"; - run "mv ~/sherlodoc $(opam config var bin)/sherlodoc"; - run ~network:Misc.network "sudo apt install -y jq"; - run ~network:Misc.network "git clone https://github.com/jonludlam/docs-ci-scripts.git && echo %s" Config.random; - (* obtain the compiled dependencies, prep folder and extract it *) - ] @ [ run ~network:Misc.network ~cache:compile_caches ~secrets:Config.Ssh.secrets "%s" @@ Misc.Cmd.list - ((Fmt.str "ssh -MNf %s" (Config.Ssh.host ssh)) :: - (* "rm -rf /home/opam/.cache/compile/*" :: *) - (common @ [ - Fmt.str - "time opam exec -- /home/opam/odoc_driver_voodoo --verbose --odoc /home/opam/odoc --odoc-md /home/opam/odoc-md --odoc-dir %a/compile --odocl-dir linked --html-dir %s --stats %s %s %s" - Fpath.pp (Storage.Base.generation_folder generation) (Fpath.to_string (Storage.Base.folder (HtmlRaw generation))) - name - (match jobty with CompileAndLink -> "" | CompileOnly -> "--actions compile-only" | LinkOnly -> "--actions link-and-gen") - (match blessing with Blessed -> "--blessed" | Universe -> ""); - (* Fmt.str "/home/opam/odoc support-files -o %s" - (Fpath.to_string (Storage.Base.folder (HtmlRaw generation))); *) - Fmt.str "jq . driver-benchmarks.json || true";] - @ (match jobty with LinkOnly -> [] | _ -> post_compile) - @ (match jobty with CompileOnly -> [] | _ -> post_link_and_html))) ]) + workdir "/home/opam/docs/"; + run "sudo chown opam:opam . "; + (* Import odoc and voodoo-do *) + copy ~from:(`Build "tools") [ "/home/opam/odoc" ] ~dst:"/home/opam/"; + copy ~from:(`Build "odoc_driver") + [ + "/home/opam/odoc_driver_voodoo"; + "/home/opam/sherlodoc"; + "/home/opam/odoc-md"; + ] + ~dst:"/home/opam/"; + run "mv ~/sherlodoc $(opam config var bin)/sherlodoc"; + run ~network:Misc.network "sudo apt install -y jq"; + run ~network:Misc.network + "git clone https://github.com/jonludlam/docs-ci-scripts.git && \ + echo %s" + Config.random; + (* obtain the compiled dependencies, prep folder and extract it *) + ] + @ [ + run ~network:Misc.network ~cache:compile_caches + ~secrets:Config.Ssh.secrets "%s" + @@ Misc.Cmd.list + (Fmt.str "ssh -MNf %s" (Config.Ssh.host ssh) + :: (* "rm -rf /home/opam/.cache/compile/*" :: *) + (common + @ [ + Fmt.str + "time opam exec -- /home/opam/odoc_driver_voodoo \ + --verbose --odoc /home/opam/odoc --odoc-md \ + /home/opam/odoc-md --odoc-dir %a/compile --odocl-dir \ + linked --html-dir %s --stats %s %s %s" + Fpath.pp + (Storage.Base.generation_folder generation) + (Fpath.to_string + (Storage.Base.folder (HtmlRaw generation))) + name + (match jobty with + | CompileAndLink -> "" + | CompileOnly -> "--actions compile-only" + | LinkOnly -> "--actions link-and-gen") + (match blessing with + | Blessed -> "--blessed" + | Universe -> ""); + (* Fmt.str "/home/opam/odoc support-files -o %s" + (Fpath.to_string (Storage.Base.folder (HtmlRaw generation))); *) + Fmt.str "jq . driver-benchmarks.json || true"; + ] + @ (match jobty with LinkOnly -> [] | _ -> post_compile) + @ + match jobty with + | CompileOnly -> [] + | _ -> post_link_and_html)); + ]) let spec_failure ~ssh ~base ~config ~blessing ~generation prep = let open Obuilder_spec in let package = Prep.package prep in let prep_folder = Storage.folder Prep package in - let compile_folder = Storage.folder (Compile (generation, blessing)) package in + let compile_folder = + Storage.folder (Compile (generation, blessing)) package + in let linked_folder = Storage.folder (Linked (generation, blessing)) package in let opam = Package.opam package in let name = OpamPackage.name_to_string opam in @@ -144,7 +200,12 @@ let spec_failure ~ssh ~base ~config ~blessing ~generation prep = run "sudo chown opam:opam . "; (* Import odoc and voodoo-do *) copy ~from:(`Build "tools") - [ "/home/opam/odoc"; "/home/opam/odoc_driver_voodoo"; "/home/opam/sherlodoc"; "/home/opam/odoc-md" ] + [ + "/home/opam/odoc"; + "/home/opam/odoc_driver_voodoo"; + "/home/opam/sherlodoc"; + "/home/opam/odoc-md"; + ] ~dst:"/home/opam/"; run "mv ~/odoc $(opam config var bin)/odoc"; run "mv ~/sherlodoc $(opam config var bin)/sherlodoc"; @@ -269,20 +330,29 @@ module Compile = struct blessing : Package.Blessing.t; } - let key { config = _; odoc; sherlodoc; deps; prep; blessing; base = _; odoc_driver_base = _; jobty } = + let key + { + config = _; + odoc; + sherlodoc; + deps; + prep; + blessing; + base = _; + odoc_driver_base = _; + jobty; + } = let odoc_sherl_hash = - Fmt.str "%s-%s" odoc sherlodoc |> Digest.string |> Digest.to_hex in - Fmt.str "v10-%s-%s-%s-%s-%a-%a" - odoc_sherl_hash + Fmt.str "%s-%s" odoc sherlodoc |> Digest.string |> Digest.to_hex + in + Fmt.str "v10-%s-%s-%s-%s-%a-%a" odoc_sherl_hash (Package.Blessing.to_string blessing) (Prep.package prep |> Package.digest) (Prep.hash prep) Fmt.( list (fun f { hashes = { compile_hash; _ }; _ } -> Fmt.pf f "%a" (Fmt.option string) compile_hash)) - deps - pp_jobty - jobty + deps pp_jobty jobty let digest t = key t |> Digest.string |> Digest.to_hex end @@ -293,7 +363,18 @@ module Compile = struct let auto_cancel = true let build { generation; _ } job - Key.{ deps; prep; blessing; config; base; odoc_driver_base; odoc; sherlodoc; jobty } = + Key. + { + deps; + prep; + blessing; + config; + base; + odoc_driver_base; + odoc; + sherlodoc; + jobty; + } = let open Lwt.Syntax in let ( let** ) = Lwt_result.bind in let package = Prep.package prep in @@ -306,7 +387,8 @@ module Compile = struct | Success -> Lwt.return_ok (spec_success ~generation ~ssh:(Config.ssh config) ~config ~base - ~odoc_driver_base ~odoc_pin ~sherlodoc_pin ~deps ~blessing ~jobty prep) + ~odoc_driver_base ~odoc_pin ~sherlodoc_pin ~deps ~blessing ~jobty + prep) | Failed -> Lwt.return_ok (spec_failure ~generation ~ssh:(Config.ssh config) ~config ~base @@ -357,7 +439,8 @@ module Compile = struct let compile = Option.map h compile in let linked = Option.map h linked in let html_raw = Option.map h html_raw in - Lwt.return_ok { compile_hash = compile; linked_hash = linked; html_hash=html_raw } + Lwt.return_ok + { compile_hash = compile; linked_hash = linked; html_hash = html_raw } with Invalid_argument _ -> Lwt.return_error (`Msg "Compile: failed to parse output") end @@ -367,16 +450,29 @@ module CompileCache = Current_cache.Make (Compile) let v ~generation ~config ~name ~blessing ~deps ~jobty prep = let open Current.Syntax in Current.component "do %s" name - |> let> prep and> blessing and> deps + |> let> prep + and> blessing + and> deps and> odoc_driver_base = Misc.default_base_image in let package = Prep.package prep in let base = Prep.base prep in - (Logs.debug (fun m -> m "Prep base: %s" (Spec.to_spec base))); + Logs.debug (fun m -> m "Prep base: %s" (Spec.to_spec base)); let odoc = Config.odoc config in - let sherlodoc = Config.sherlodoc config in + let sherlodoc = Config.sherlodoc config in let output = CompileCache.get { Compile.generation } - Compile.Key.{ prep; blessing; deps; config; base; odoc_driver_base; odoc; sherlodoc; jobty } + Compile.Key. + { + prep; + blessing; + deps; + config; + base; + odoc_driver_base; + odoc; + sherlodoc; + jobty; + } in Current.Primitive.map_result (Result.map (fun hashes -> { package; blessing; hashes })) File "test/monitor/test_monitor.ml", line 1, characters 0-0: diff --git a/_build/default/test/monitor/test_monitor.ml b/_build/default/test/monitor/.formatted/test_monitor.ml index 087306c..b8fb883 100644 --- a/_build/default/test/monitor/test_monitor.ml +++ b/_build/default/test/monitor/.formatted/test_monitor.ml @@ -19,7 +19,10 @@ let running = let fakepkg ~blessing name = let open Docs_ci_lib in let root = OpamPackage.of_string name in - let pkg = Package.make ~ocaml_version:Ocaml_version.Releases.latest ~blacklist:[] ~commit:"0" ~root [] in + let pkg = + Package.make ~ocaml_version:Ocaml_version.Releases.latest ~blacklist:[] + ~commit:"0" ~root [] + in let blessing = let set = Package.Blessing.Set.v ~counts:(Package.Map.singleton pkg 0) [ pkg ] File "src/lib/monitor.ml", line 1, characters 0-0: diff --git a/_build/default/src/lib/monitor.ml b/_build/default/src/lib/.formatted/monitor.ml index 56672bf..e103b37 100644 --- a/_build/default/src/lib/monitor.ml +++ b/_build/default/src/lib/.formatted/monitor.ml @@ -86,16 +86,18 @@ let render_level = let render_list ~bullet ~level ~items ~render = let open Tyxml_html in - Some (ul - (List.filter_map - (fun (name, item) -> - match render ~level:(level + 1) item with - | None -> None - | Some elt -> - Some (li - ~a:[ a_style ("list-style-type: " ^ bullet) ] - [ (render_level level) [ txt name ]; elt ])) - items)) + Some + (ul + (List.filter_map + (fun (name, item) -> + match render ~level:(level + 1) item with + | None -> None + | Some elt -> + Some + (li + ~a:[ a_style ("list-style-type: " ^ bullet) ] + [ (render_level level) [ txt name ]; elt ])) + items)) let rec render ~level = let open Tyxml_html in @@ -113,7 +115,8 @@ let rec render ~level = fun v -> Some (a ~a:[ a_href ("/job/" ^ job_id) ] [ txt v ]) | _ -> fun x -> Some (txt x) with (* if current is not a primitive term *) - | Failure _ -> (fun x -> Some (txt x)) + | Failure _ -> + fun x -> Some (txt x) in match result with | Error (`Msg msg) -> container ("error: " ^ msg) @@ -136,10 +139,14 @@ let get_opam_package_info t opam_package = in match Package.Blessing.Set.blessed blessing_set with | None -> - let preps = Package.Map.filter (fun package _ -> - Package.opam package = opam_package) t.preps in - Or (Package.Map.bindings preps |> List.map (fun (package, tree) -> - (Package.id package, tree))) + let preps = + Package.Map.filter + (fun package _ -> Package.opam package = opam_package) + t.preps + in + Or + (Package.Map.bindings preps + |> List.map (fun (package, tree) -> (Package.id package, tree))) | Some blessed_package -> let blessed_pipeline = Package.Map.find blessed_package t.trees in blessed_pipeline @@ -179,7 +186,11 @@ let rec to_steps description arr = function let render_package_state t opam_package = let name = OpamPackage.name_to_string opam_package in - let all = Package.Map.filter_map (fun p x -> if Package.opam p = opam_package then Some x else None) t.preps in + let all = + Package.Map.filter_map + (fun p x -> if Package.opam p = opam_package then Some x else None) + t.preps + in match OpamPackage.Map.find_opt opam_package t.solve_failures with | Some reason -> let open Tyxml_html in @@ -193,14 +204,15 @@ let render_package_state t opam_package = let* blessed_pipeline = get_opam_package_info t opam_package in let open Tyxml_html in Ok - ([ - h1 [ txt ("Package " ^ name) ]] @ - (render ~level:1 (simplify blessed_pipeline) |> Option.to_list) - @ [h2 [ txt ("All universes")]] - - @ List.flatten (List.map (fun (pkg, pipeline) -> - ([ h3 [ txt (Package.universe pkg |> Package.Universe.hash) ] ] - @ (render ~level:2 (simplify pipeline) |> Option.to_list))) (Package.Map.bindings all))) + ([ h1 [ txt ("Package " ^ name) ] ] + @ (render ~level:1 (simplify blessed_pipeline) |> Option.to_list) + @ [ h2 [ txt "All universes" ] ] + @ List.flatten + (List.map + (fun (pkg, pipeline) -> + [ h3 [ txt (Package.universe pkg |> Package.Universe.hash) ] ] + @ (render ~level:2 (simplify pipeline) |> Option.to_list)) + (Package.Map.bindings all))) let handle t ~engine:_ str = object @@ -270,33 +282,44 @@ let lookup_failed_compiles t = let error_active = ref 0 in let error_blocked = ref 0 in let ok = ref 0 in - let x = Package.Map.fold (fun _pkg current acc -> - let status : step_status = - match Current.observe current with - | Error (`Msg msg) -> incr error_msg; Err msg - | Error (`Active _) -> incr error_active; Active - | Error `Blocked -> incr error_blocked; Blocked - | Ok _ -> incr ok; OK - in - let container = - try - Current.Analysis.metadata current - |> Current.observe - |> Result.to_option - |> Option.join - |> function - | Some { job_id = Some job_id; _ } -> - fun v -> a ~a:[ a_href ("/job/" ^ job_id) ] [ txt v ] - | _ -> txt - with (* if current is not a primitive term *) - | Failure _ -> txt - in - match status with - | Err msg -> (container (Printf.sprintf "error: %s" msg)) :: acc - | _ -> acc) - t.html [] in + let x = + Package.Map.fold + (fun _pkg current acc -> + let status : step_status = + match Current.observe current with + | Error (`Msg msg) -> + incr error_msg; + Err msg + | Error (`Active _) -> + incr error_active; + Active + | Error `Blocked -> + incr error_blocked; + Blocked + | Ok _ -> + incr ok; + OK + in + let container = + try + Current.Analysis.metadata current + |> Current.observe + |> Result.to_option + |> Option.join + |> function + | Some { job_id = Some job_id; _ } -> + fun v -> a ~a:[ a_href ("/job/" ^ job_id) ] [ txt v ] + | _ -> txt + with (* if current is not a primitive term *) + | Failure _ -> txt + in + match status with + | Err msg -> container (Printf.sprintf "error: %s" msg) :: acc + | _ -> acc) + t.html [] + in (x, !error_active, !error_blocked, !error_msg, !ok) - + let lookup_solve_failures t = OpamPackage.Map.keys t.solve_failures |> List.map (fun k -> (k, Failed)) @@ -376,12 +399,11 @@ let render_passing_packages t = let render_package_root t = let max_version = map_max_versions t in let failed, pending = lookup_failed_pending t in - let (failed_do, n1, n2, n3, n4) = lookup_failed_compiles t in + let failed_do, n1, n2, n3, n4 = lookup_failed_compiles t in let open Tyxml_html in [ - h1 [ txt (Printf.sprintf "Failed 'do' jobs (%d, %d, %d, %d)" n1 n2 n3 n4)]; - ul (List.map (fun x -> - li [x]) failed_do); + h1 [ txt (Printf.sprintf "Failed 'do' jobs (%d, %d, %d, %d)" n1 n2 n3 n4) ]; + ul (List.map (fun x -> li [ x ]) failed_do); h1 [ txt "Failed packages" ]; p [ txt ("total: " ^ Int.to_string (List.length failed)) ]; ul File "src/lib/prep.ml", line 1, characters 0-0: diff --git a/_build/default/src/lib/prep.ml b/_build/default/src/lib/.formatted/prep.ml index 5dfd5bd..0f45f07 100644 --- a/_build/default/src/lib/prep.ml +++ b/_build/default/src/lib/.formatted/prep.ml @@ -15,16 +15,17 @@ module OpamPackage = struct | _ -> Error "failed to run version" end -let opam_cache_version="v1" +let opam_cache_version = "v1" + module Cache = struct let fname id pkg = let name = OpamPackage.to_string pkg in let fname = name ^ "-opam.tar.bz2" in - Fpath.(Current.state_dir id / name / fname ) + Fpath.(Current.state_dir id / name / fname) let id = "opam-files-cache-" ^ opam_cache_version - type cache_value = (bool * string) + type cache_value = bool * string let fname = fname id @@ -34,8 +35,11 @@ module Cache = struct | Ok true -> true | Ok false -> false | Error (`Msg m) -> - Current.Job.log job "Error checking for opam files cache for package %s: %s" (OpamPackage.to_string pkg) m; - false + Current.Job.log job + "Error checking for opam files cache for package %s: %s" + (OpamPackage.to_string pkg) + m; + false let write ((pkg, value) : OpamPackage.t * cache_value) = let fname = fname pkg in @@ -51,111 +55,128 @@ module Cache = struct let result = Marshal.from_channel file in close_in file; Some result - with Failure _ | Sys_error _ as e -> - Current.Job.log job "Error reading opam files cache for package %s: %s" (OpamPackage.to_string pkg) (Printexc.to_string e); + with (Failure _ | Sys_error _) as e -> + Current.Job.log job "Error reading opam files cache for package %s: %s" + (OpamPackage.to_string pkg) + (Printexc.to_string e); None end - module OpamFiles = struct - type t = No_context let id = "opam-files" - let auto_cancel = true module Key = struct type t = { repo : Current_git.Commit.t; packages : OpamPackage.t list } - + let digest { repo; packages } = - Current_git.Commit.hash repo ^ String.concat ";" (List.map OpamPackage.to_string packages) + Current_git.Commit.hash repo + ^ String.concat ";" (List.map OpamPackage.to_string packages) end module Value = struct - type ('a, 'b) r = ('a, 'b) result = - | Ok of 'a - | Error of 'b [@@deriving yojson] + type ('a, 'b) r = ('a, 'b) result = Ok of 'a | Error of 'b + [@@deriving yojson] - type serialisable = (OpamPackage.t * ((bool * string), [`Msg of string]) r) list - [@@deriving yojson] + type serialisable = + (OpamPackage.t * (bool * string, [ `Msg of string ]) r) list + [@@deriving yojson] - type t = ((bool * string), [`Msg of string]) r OpamPackage.Map.t + type t = (bool * string, [ `Msg of string ]) r OpamPackage.Map.t let to_yojson t = t |> OpamPackage.Map.bindings |> serialisable_to_yojson - let of_yojson t = t |> serialisable_of_yojson |> Result.map OpamPackage.Map.of_list - let marshal t = t |> to_yojson |> Yojson.Safe.to_string - let unmarshal t = t |> Yojson.Safe.from_string |> of_yojson |> Result.get_ok + let of_yojson t = + t |> serialisable_of_yojson |> Result.map OpamPackage.Map.of_list + + let marshal t = t |> to_yojson |> Yojson.Safe.to_string + let unmarshal t = t |> Yojson.Safe.from_string |> of_yojson |> Result.get_ok end let pool = Current.Pool.create ~label:"git_pool" 5 + let build _ job { Key.repo; packages } = + let package_path pkg = + Fpath.( + v "packages" + / (OpamPackage.name pkg |> OpamPackage.Name.to_string) + / OpamPackage.to_string pkg) + in + let open Lwt.Infix in + Current.Job.start ~pool ~level:Harmless job >>= fun () -> + let to_do = List.filter (fun x -> not (Cache.mem job x)) packages in + Current_git.with_checkout ~job repo (fun path -> + Current.Process.with_tmpdir ~prefix:"opam-files" (fun fp -> + let open Lwt.Infix in + let r = + Lwt_list.map_s + (fun pkg -> + let open Lwt.Syntax in + let tarfile = + Fpath.( + fp / Fmt.str "%s-opam.tar.bz2" (OpamPackage.to_string pkg)) + in + let has_depext = + match + Bos.OS.File.read Fpath.(path // package_path pkg / "opam") + with + | Ok content -> + let f = OpamFile.OPAM.read_from_string content in + let _x = OpamFile.OPAM.depends f in + OpamFile.OPAM.depexts f <> [] + | _ -> false + in + let command = + Bos.Cmd.( + v "tar" + % "-C" + % Fpath.to_string path + % "-jcf" + % Fpath.to_string tarfile + % Fpath.to_string (package_path pkg)) + in + Current.Job.log job "About to exec"; + let* res = + Current.Process.exec ~cancellable:true ~job + ("", Bos.Cmd.to_list command |> Array.of_list) + in + Current.Job.log job "Finished"; + match res with + | Ok () -> ( + let res = Bos.OS.File.read tarfile in + match res with + | Ok contents -> + Cache.write + (pkg, (has_depext, Base64.encode_string contents)); + Lwt.return (pkg, Ok ()) + | Error (`Msg m) -> Lwt.return (pkg, Error (`Msg m))) + | Error (`Msg m) -> Lwt.return (pkg, Error (`Msg m))) + to_do + in + r >>= fun results -> + let results_map = OpamPackage.Map.of_list results in + List.map + (fun x -> + match Cache.read job x with + | Some v -> (x, Ok v) + | None -> ( + match OpamPackage.Map.find x results_map with + | Error (`Msg m) -> (x, Error (`Msg m)) + | Ok () -> + (x, Error (`Msg "Failed to read cache after writing")) + | exception Not_found -> + (x, Error (`Msg "Didn't even try to solve for package")) + )) + packages + |> OpamPackage.Map.of_list + |> Lwt.return_ok)) - -let build _ job { Key.repo; packages } = - let package_path pkg = - Fpath.(v "packages" / (OpamPackage.name pkg |> OpamPackage.Name.to_string) / OpamPackage.to_string pkg) - in - let open Lwt.Infix in - Current.Job.start ~pool ~level:Harmless job >>= fun () -> - let to_do = List.filter (fun x -> not (Cache.mem job x)) packages in - Current_git.with_checkout ~job repo (fun path -> - Current.Process.with_tmpdir ~prefix:"opam-files" (fun fp -> - let open Lwt.Infix in - let r = Lwt_list.map_s (fun pkg -> - let open Lwt.Syntax in - let tarfile = Fpath.(fp / (Fmt.str "%s-opam.tar.bz2" (OpamPackage.to_string pkg))) in - let has_depext = - match Bos.OS.File.read Fpath.(path // package_path pkg / "opam") with - | Ok content -> - let f = OpamFile.OPAM.read_from_string content in - let _x = OpamFile.OPAM.depends f in - OpamFile.OPAM.depexts f <> [] - | _ -> false - in - let command = - Bos.Cmd.( - v "tar" - % "-C" - % Fpath.to_string path - % "-jcf" - % Fpath.to_string tarfile - % (Fpath.to_string (package_path pkg) ) - ) - in - Current.Job.log job "About to exec"; - let* res = Current.Process.exec ~cancellable:true ~job - ("", Bos.Cmd.to_list command |> Array.of_list) in - Current.Job.log job "Finished"; - match res with - | Ok () -> begin - let res = Bos.OS.File.read tarfile in - match res with - | Ok contents -> - Cache.write (pkg, (has_depext, Base64.encode_string contents)); - Lwt.return (pkg, Ok ()) - | Error (`Msg m) -> Lwt.return (pkg, Error (`Msg m)) - end - | Error (`Msg m) -> Lwt.return (pkg, Error (`Msg m)) - ) to_do in - r >>= fun results -> - let results_map = OpamPackage.Map.of_list results in - List.map (fun x -> - match Cache.read job x with - | Some v -> (x, Ok v) - | None -> begin - match OpamPackage.Map.find x results_map with - | Error (`Msg m) -> (x, Error (`Msg m)) - | Ok () -> (x, Error (`Msg "Failed to read cache after writing")) - | exception Not_found -> - (x, Error (`Msg "Didn't even try to solve for package")) - end) packages |> OpamPackage.Map.of_list |> Lwt.return_ok - )) let pp f { Key.repo; packages } = Fmt.pf f "opamfiles\n%a\n%a" Current_git.Commit.pp_short repo - (Fmt.list (Fmt.of_to_string OpamPackage.to_string)) packages - + (Fmt.list (Fmt.of_to_string OpamPackage.to_string)) + packages end module type F = Current_cache.S.BUILDER @@ -177,41 +198,50 @@ let not_base x = ]) let add_base ocaml_version init = - let add_one x pkgs = - if List.mem x pkgs then pkgs else x :: pkgs - in + let add_one x pkgs = if List.mem x pkgs then pkgs else x :: pkgs in let mk n v = OpamPackage.create (OpamPackage.Name.of_string n) (OpamPackage.Version.of_string v) in let v = ocaml_version in - let std = List.fold_right add_one [ - mk "conf-graphviz" "0.1"; - mk "conf-which" "1"; - mk "base-unix" "base"; - mk "base-bigarray" "base"; - mk "base-threads" "base"; - mk "ocaml-base-compiler" (Ocaml_version.to_string ocaml_version); - mk "ocaml" (Ocaml_version.to_string ocaml_version); - ] init in - let extra = List.assoc (Ocaml_version.major v) [ - 5, [mk "base-domains" "base"; - mk "base-nnp" "base"; - mk "base-effects" "base"; - mk "host-arch-x86_64" "1"; - mk "host-system-other" "1"; - mk "ocaml-options-vanilla" "1"; - ]; - 4, [ - mk "ocaml-options-vanilla" "1"; - mk "host-system-other" "1"; - mk "host-arch-x86_64" "1"; - ] - - ] in + let std = + List.fold_right add_one + [ + mk "conf-graphviz" "0.1"; + mk "conf-which" "1"; + mk "base-unix" "base"; + mk "base-bigarray" "base"; + mk "base-threads" "base"; + mk "ocaml-base-compiler" (Ocaml_version.to_string ocaml_version); + mk "ocaml" (Ocaml_version.to_string ocaml_version); + ] + init + in + let extra = + List.assoc (Ocaml_version.major v) + [ + ( 5, + [ + mk "base-domains" "base"; + mk "base-nnp" "base"; + mk "base-effects" "base"; + mk "host-arch-x86_64" "1"; + mk "host-system-other" "1"; + mk "ocaml-options-vanilla" "1"; + ] ); + ( 4, + [ + mk "ocaml-options-vanilla" "1"; + mk "host-system-other" "1"; + mk "host-arch-x86_64" "1"; + ] ); + ] + in let ocaml_config = - if Ocaml_version.major v = 5 then [ mk "ocaml-config" "3"] else if Ocaml_version.minor v >= 12 then [ mk "ocaml-config" "2"] else [ mk "ocaml-config" "1" ] + if Ocaml_version.major v = 5 then [ mk "ocaml-config" "3" ] + else if Ocaml_version.minor v >= 12 then [ mk "ocaml-config" "2" ] + else [ mk "ocaml-config" "1" ] in ocaml_config @ std @ extra @@ -232,12 +262,13 @@ let spec ~ssh ~tools_base ~base ~opamfiles (prep : Package.t) = let ocaml_version = Package.ocaml_version prep in let prep_caches = - [Obuilder_spec.Cache.v "prep-cache" ~target:"/home/opam/.cache/prep"] in + [ Obuilder_spec.Cache.v "prep-cache" ~target:"/home/opam/.cache/prep" ] + in let cache = cache @ prep_caches in let packages_topo_list = - all_deps + all_deps |> Package.topo_sort |> List.filter (fun x -> Package.opam x |> not_base) in @@ -253,19 +284,17 @@ let spec ~ssh ~tools_base ~base ~opamfiles (prep : Package.t) = ### output ### # Error: link: /home/opam/.cache/dune/db/v2/temp/promoting: Invalid cross-device link *) - - let prep_storage_folder = (Storage.Prep0, prep) in let prep_folder = Storage.folder Prep0 prep in let create_dir_and_copy_logs_if_not_exist = let command = Fmt.str - "([ -d $1 ] || (echo FAILED:$2 && mkdir -p $1 && cp ~/opam.err.log \ - $1 && opam show $3 --raw > $1/opam)) && (%s)" + "([ -d $1 ] || (echo FAILED:$2 && mkdir -p $1 && cp ~/opam.err.log $1 \ + && opam show $3 --raw > $1/opam)) && (%s)" (Misc.tar_cmd (Fpath.v "$1")) in - Storage.for_all [prep_storage_folder] command + Storage.for_all [ prep_storage_folder ] command in let install_opamfiles pkg tar_b64 = @@ -273,79 +302,111 @@ let spec ~ssh ~tools_base ~base ~opamfiles (prep : Package.t) = Fmt.str "echo %s ; echo %s | base64 -d | sudo tar -jxC /src" name tar_b64 in - let install_all_opamfiles = List.filter_map (fun pkg -> - match OpamPackage.Map.find (Package.opam pkg) opamfiles with - | Ok (_, opamfiles) -> - Some (install_opamfiles (Package.opam pkg) opamfiles) - | Error (`Msg _m) -> - None - | exception Not_found -> - None) packages_topo_list + let install_all_opamfiles = + List.filter_map + (fun pkg -> + match OpamPackage.Map.find (Package.opam pkg) opamfiles with + | Ok (_, opamfiles) -> + Some (install_opamfiles (Package.opam pkg) opamfiles) + | Error (`Msg _m) -> None + | exception Not_found -> None) + packages_topo_list in - let install_cmds = List.map (run ~network "%s") (Misc.Cmd.list_list install_all_opamfiles) in + let install_cmds = + List.map (run ~network "%s") (Misc.Cmd.list_list install_all_opamfiles) + in let install_packages = - "echo \"START OF INSTALL PACKAGES\" && date" :: ( - List.flatten @@ List.map (fun pkg -> - match OpamPackage.Map.find (Package.opam pkg) opamfiles with - | Ok (_has_depext, _opamfiles) -> - [ Fmt.str "time ~/docs/docs-ci-scripts/download_prep.sh %s %s %s %s" - (Config.Ssh.host ssh) - (Config.Ssh.storage_folder ssh) - (Fpath.to_string (Storage.folder Storage.Prep0 pkg)) - (if Package.should_cache pkg then "true" else "false") - ] - | Error _ | exception _ -> - [ Fmt.str "echo Failed to find opamfiles for %s" - (Package.opam pkg |> OpamPackage.to_string) - ]) packages_topo_list) + "echo \"START OF INSTALL PACKAGES\" && date" + :: (List.flatten + @@ List.map + (fun pkg -> + match OpamPackage.Map.find (Package.opam pkg) opamfiles with + | Ok (_has_depext, _opamfiles) -> + [ + Fmt.str + "time ~/docs/docs-ci-scripts/download_prep.sh %s %s %s %s" + (Config.Ssh.host ssh) + (Config.Ssh.storage_folder ssh) + (Fpath.to_string (Storage.folder Storage.Prep0 pkg)) + (if Package.should_cache pkg then "true" else "false"); + ] + | Error _ | (exception _) -> + [ + Fmt.str "echo Failed to find opamfiles for %s" + (Package.opam pkg |> OpamPackage.to_string); + ]) + packages_topo_list) in - let any_depexts = List.exists (fun pkg -> match OpamPackage.Map.find (Package.opam pkg) opamfiles with | Ok (has_depext, _) -> has_depext | Error _ | exception _ -> false) packages_topo_list in + let any_depexts = + List.exists + (fun pkg -> + match OpamPackage.Map.find (Package.opam pkg) opamfiles with + | Ok (has_depext, _) -> has_depext + | Error _ | (exception _) -> false) + packages_topo_list + in let extra_opamfiles = - List.flatten (List.map (fun pkg -> - match OpamPackage.Map.find pkg opamfiles with - | Ok (_has_depext, opamfiles) -> - [ install_opamfiles pkg opamfiles ] - | _ | exception _ -> [ Fmt.str "echo Missing %s" (OpamPackage.to_string pkg)]) (add_base ocaml_version []) - ) + List.flatten + (List.map + (fun pkg -> + match OpamPackage.Map.find pkg opamfiles with + | Ok (_has_depext, opamfiles) -> [ install_opamfiles pkg opamfiles ] + | _ | (exception _) -> + [ Fmt.str "echo Missing %s" (OpamPackage.to_string pkg) ]) + (add_base ocaml_version [])) in let pkg_opamfile = - (match OpamPackage.Map.find (Package.opam prep) opamfiles with - | Ok (_has_depext, opamfiles) -> + match OpamPackage.Map.find (Package.opam prep) opamfiles with + | Ok (_has_depext, opamfiles) -> [ install_opamfiles (Package.opam prep) opamfiles ] - | Error _ | exception _ -> []) + | Error _ | (exception _) -> [] in let post_steps = - [ "echo \"START OF BUILD PACKAGE\" && date"; - - "opam update && /home/opam/opamh make-state --output $(opam var prefix)/.opam-switch/switch-state"; - if any_depexts then Fmt.str "opam depext %s" (OpamPackage.to_string (Package.opam prep)) else "echo no depexts"; - Fmt.str "(opam update && opam install -vv --debug-level=2 --confirm-level=unsafe-yes --solver=builtin-0install %s 2>&1 && opam clean -s | tee ~/opam.err.log) || echo \ - 'Failed to install all packages'" (Package.opam prep |> OpamPackage.to_string); + [ + "echo \"START OF BUILD PACKAGE\" && date"; + "opam update && /home/opam/opamh make-state --output $(opam var \ + prefix)/.opam-switch/switch-state"; + (if any_depexts then + Fmt.str "opam depext %s" (OpamPackage.to_string (Package.opam prep)) + else "echo no depexts"); + Fmt.str + "(opam update && opam install -vv --debug-level=2 \ + --confirm-level=unsafe-yes --solver=builtin-0install %s 2>&1 && opam \ + clean -s | tee ~/opam.err.log) || echo 'Failed to install all \ + packages'" + (Package.opam prep |> OpamPackage.to_string); "echo \"END OF BUILD PACKAGE\" && date"; Fmt.str "mkdir -p %s" (Fpath.to_string prep_folder); - Fmt.str "/home/opam/opamh save --output=%s/content.tar %s" (Fpath.to_string prep_folder) (Package.opam prep |> OpamPackage.name |> OpamPackage.Name.to_string); + Fmt.str "/home/opam/opamh save --output=%s/content.tar %s" + (Fpath.to_string prep_folder) + (Package.opam prep |> OpamPackage.name |> OpamPackage.Name.to_string); Fmt.str "time rsync -aR ./%s %s:%s/." (Fpath.to_string prep_folder) (Config.Ssh.host ssh) (Config.Ssh.storage_folder ssh); "rm -f /tmp/*.tar"; "rm -rf $(opam var prefix)"; - ] in + ] + in - let persistent_ssh = [ Fmt.str "/usr/bin/time -f \"SSH time %%E\" ssh -MNf %s" (Config.Ssh.host ssh) ] in + let persistent_ssh = + [ + Fmt.str "/usr/bin/time -f \"SSH time %%E\" ssh -MNf %s" + (Config.Ssh.host ssh); + ] + in let copy_results = - (create_dir_and_copy_logs_if_not_exist) - @ - (* Compute hashes *) - (Storage.for_all [prep_storage_folder] - (Storage.Tar.hash_command ~prefix:"HASHES" ())) + create_dir_and_copy_logs_if_not_exist + @ (* Compute hashes *) + Storage.for_all [ prep_storage_folder ] + (Storage.Tar.hash_command ~prefix:"HASHES" ()) in (* let install_packages = persistent_ssh @ pkg_opamfile @ extra_opamfiles @ install_packages @ post_steps @ copy_results in *) @@ -354,39 +415,53 @@ let spec ~ssh ~tools_base ~base ~opamfiles (prep : Package.t) = |> Spec.children ~name:"tools" tools |> Spec.add ([ - (* Install required packages *) - run "sudo mkdir /src"; - run "echo b3BhbS12ZXJzaW9uOiAiMi4wIgpicm93c2U6ICJodHRwczovL29wYW0ub2NhbWwub3JnL3BrZy8iCnVwc3RyZWFtOiAiaHR0cHM6Ly9naXRodWIuY29tL29jYW1sL29wYW0tcmVwb3NpdG9yeS90cmVlL21hc3Rlci8iCmFubm91bmNlOiBbCiIiIgpbV0FSTklOR10gb3BhbSBpcyBvdXQtb2YtZGF0ZS4gUGxlYXNlIGNvbnNpZGVyIHVwZGF0aW5nIGl0IChodHRwczovL29wYW0ub2NhbWwub3JnL2RvYy9JbnN0YWxsLmh0bWwpCiIiIiB7KG9wYW0tdmVyc2lvbiA+PSAiMi4xLjB+fiIgJiBvcGFtLXZlcnNpb24gPCAiMi4xLjUiKSB8IG9wYW0tdmVyc2lvbiA8ICIyLjAuMTAifQoiIiIKW0lORk9dIG9wYW0gMi4xIGluY2x1ZGVzIG1hbnkgcGVyZm9ybWFuY2UgaW1wcm92ZW1lbnRzIG92ZXIgMi4wOyBwbGVhc2UgY29uc2lkZXIgdXBncmFkaW5nIChodHRwczovL29wYW0ub2NhbWwub3JnL2RvYy9JbnN0YWxsLmh0bWwpCiIiIiB7b3BhbS12ZXJzaW9uID49ICIyLjAuMTAiICYgb3BhbS12ZXJzaW9uIDwgIjIuMS4wfn4ifQpdCg== | base64 -d | sudo tee /src/repo"; - run "ls -lR /src"; - (* Re-initialise opam after switching from opam.2.0 to 2.3. *) - run ~network - "sudo ln -f /usr/bin/opam-2.3 /usr/bin/opam && opam repo remove default --all && opam init --reinit \ - -ni"; - run "sudo mkdir /src/packages"; - run "opam repo add opam /src"; - copy ~from:(`Build "tools") - [ "/home/opam/opamh" ] - ~dst:"/home/opam/"; - (* Enable build cache conditionally on dune version *) - env "DUNE_CACHE" "disabled"; - run "mkdir /home/opam/docs"; - (* Pre-install some of the most popular packages *) - run ~network:Misc.network "sudo apt-get update && sudo apt-get install -qq -yy pkg-config libgmp-dev libev-dev libssl-dev zlib1g-dev libpcre3-dev libffi-dev m4 xdot autoconf libsqlite3-dev cmake libcurl4-gnutls-dev libpcre2-dev libsdl2-dev time python3 libexpat1-dev libcairo2-dev"; - - run ~network:Misc.network "git clone https://github.com/jonludlam/docs-ci-scripts.git /home/opam/docs/docs-ci-scripts && echo %s" Config.random; - - (* run ~network ~cache ~secrets:Config.Ssh.secrets "%s" @@ Misc.Cmd.list install_packages *) - ] @ install_cmds @ [ - run ~network ~cache ~secrets:Config.Ssh.secrets "%s" @@ Misc.Cmd.list (persistent_ssh @ pkg_opamfile @ extra_opamfiles @ install_packages) ] @ [ - run ~network ~cache ~secrets:Config.Ssh.secrets "%s" @@ Misc.Cmd.list (post_steps @ copy_results); - ] - ) - - + (* Install required packages *) + run "sudo mkdir /src"; + run + "echo \ + b3BhbS12ZXJzaW9uOiAiMi4wIgpicm93c2U6ICJodHRwczovL29wYW0ub2NhbWwub3JnL3BrZy8iCnVwc3RyZWFtOiAiaHR0cHM6Ly9naXRodWIuY29tL29jYW1sL29wYW0tcmVwb3NpdG9yeS90cmVlL21hc3Rlci8iCmFubm91bmNlOiBbCiIiIgpbV0FSTklOR10gb3BhbSBpcyBvdXQtb2YtZGF0ZS4gUGxlYXNlIGNvbnNpZGVyIHVwZGF0aW5nIGl0IChodHRwczovL29wYW0ub2NhbWwub3JnL2RvYy9JbnN0YWxsLmh0bWwpCiIiIiB7KG9wYW0tdmVyc2lvbiA+PSAiMi4xLjB+fiIgJiBvcGFtLXZlcnNpb24gPCAiMi4xLjUiKSB8IG9wYW0tdmVyc2lvbiA8ICIyLjAuMTAifQoiIiIKW0lORk9dIG9wYW0gMi4xIGluY2x1ZGVzIG1hbnkgcGVyZm9ybWFuY2UgaW1wcm92ZW1lbnRzIG92ZXIgMi4wOyBwbGVhc2UgY29uc2lkZXIgdXBncmFkaW5nIChodHRwczovL29wYW0ub2NhbWwub3JnL2RvYy9JbnN0YWxsLmh0bWwpCiIiIiB7b3BhbS12ZXJzaW9uID49ICIyLjAuMTAiICYgb3BhbS12ZXJzaW9uIDwgIjIuMS4wfn4ifQpdCg== \ + | base64 -d | sudo tee /src/repo"; + run "ls -lR /src"; + (* Re-initialise opam after switching from opam.2.0 to 2.3. *) + run ~network + "sudo ln -f /usr/bin/opam-2.3 /usr/bin/opam && opam repo remove \ + default --all && opam init --reinit -ni"; + run "sudo mkdir /src/packages"; + run "opam repo add opam /src"; + copy ~from:(`Build "tools") [ "/home/opam/opamh" ] ~dst:"/home/opam/"; + (* Enable build cache conditionally on dune version *) + env "DUNE_CACHE" "disabled"; + run "mkdir /home/opam/docs"; + (* Pre-install some of the most popular packages *) + run ~network:Misc.network + "sudo apt-get update && sudo apt-get install -qq -yy pkg-config \ + libgmp-dev libev-dev libssl-dev zlib1g-dev libpcre3-dev \ + libffi-dev m4 xdot autoconf libsqlite3-dev cmake \ + libcurl4-gnutls-dev libpcre2-dev libsdl2-dev time python3 \ + libexpat1-dev libcairo2-dev"; + run ~network:Misc.network + "git clone https://github.com/jonludlam/docs-ci-scripts.git \ + /home/opam/docs/docs-ci-scripts && echo %s" + Config.random; + (* run ~network ~cache ~secrets:Config.Ssh.secrets "%s" @@ Misc.Cmd.list install_packages *) + ] + @ install_cmds + @ [ + run ~network ~cache ~secrets:Config.Ssh.secrets "%s" + @@ Misc.Cmd.list + (persistent_ssh + @ pkg_opamfile + @ extra_opamfiles + @ install_packages); + ] + @ [ + run ~network ~cache ~secrets:Config.Ssh.secrets "%s" + @@ Misc.Cmd.list (post_steps @ copy_results); + ]) + type prep_result = Success | Failed -type prep_output = - { +type prep_output = { base : Spec.t; hash : string; prep_hash : string; @@ -400,10 +475,7 @@ module Prep = struct let id = "voodoo-prep2" let auto_cancel = true - module Key = struct - - type t = { prep : Package.t; base : Spec.t; @@ -412,14 +484,13 @@ module Prep = struct opamfiles : (bool * string) Current.or_error OpamPackage.Map.t; } - let digest { prep; base = _; tools_base = _; config = _; opamfiles = _; } = + let digest { prep; base = _; tools_base = _; config = _; opamfiles = _ } = (* base is derived from 'prep' so we don't need to include it in the hash *) Fmt.str "%s\n%s\n%s\n" prep_version (Package.digest prep) (Package.digest prep) end - let pp f Key.{ prep; _ } = - Fmt.pf f "Voodoo prep %a" Package.pp prep + let pp f Key.{ prep; _ } = Fmt.pf f "Voodoo prep %a" Package.pp prep module Value = struct type item = Storage.id_hash [@@deriving yojson] @@ -429,8 +500,7 @@ module Prep = struct let unmarshal t = t |> Yojson.Safe.from_string |> of_yojson |> Result.get_ok end - let build No_context job Key.{ prep; base; tools_base; config; opamfiles; } - = + let build No_context job Key.{ prep; base; tools_base; config; opamfiles } = let open Lwt.Syntax in let ( let** ) = Lwt_result.bind in (* Problem: no rebuild if the opam definition changes without affecting the universe hash. @@ -438,7 +508,9 @@ module Prep = struct requires changes in the solver. For now we rebuild only if voodoo-prep changes. *) - let spec = spec ~ssh:(Config.ssh config) ~base ~tools_base ~opamfiles prep in + let spec = + spec ~ssh:(Config.ssh config) ~base ~tools_base ~opamfiles prep + in let action = Misc.to_ocluster_submission spec in let version = Misc.cache_hint prep in Current.Job.log job "Prep job: prep %a" Package.pp prep; @@ -461,8 +533,7 @@ module Prep = struct --connect ocluster-submission.cap --cache-hint %s \\@.--secret \ ssh_privkey:id_rsa --secret ssh_pubkey:id_rsa.pub--secret \ ssh_config:ssh_config@.@." - (Spec.to_spec spec) - cache_hint); + (Spec.to_spec spec) cache_hint); Capnp_rpc_lwt.Capability.with_ref build_job @@ fun build_job -> (* extract result from logs *) @@ -521,7 +592,6 @@ end module PrepCache = Current_cache.Make (Prep) - type t = prep_output let hash t = t.hash @@ -545,20 +615,29 @@ module StringSet = Set.Make (String) let weekly = Current_cache.Schedule.v ~valid_for:(Duration.of_day 7) () let v ~config ~spec ~deps ~opamfiles ~prep = - let open Current.Syntax in + let open Current.Syntax in let opamfiles : OpamFiles.Value.t Current.t = opamfiles in Current.component "voodoo-prep %s" (prep |> Package.digest) - |> let> spec and> opamfiles and> deps - and> tools_base = Misc.default_base_image in - ignore deps; - PrepCache.get No_context { prep; config; base = spec; tools_base; opamfiles; } - |> Current.Primitive.map_result (Result.map (fun (artifacts_branches_output, _failed) -> - let _artifacts_branches_output = - artifacts_branches_output - |> List.to_seq - |> Seq.map (fun Storage.{ id; hash } -> (id, hash)) - |> StringMap.of_seq - in - - { base = spec; prep_hash = ""; result=Success; package=prep; hash="" })) - + |> let> spec + and> opamfiles + and> deps + and> tools_base = Misc.default_base_image in + ignore deps; + PrepCache.get No_context + { prep; config; base = spec; tools_base; opamfiles } + |> Current.Primitive.map_result + (Result.map (fun (artifacts_branches_output, _failed) -> + let _artifacts_branches_output = + artifacts_branches_output + |> List.to_seq + |> Seq.map (fun Storage.{ id; hash } -> (id, hash)) + |> StringMap.of_seq + in + + { + base = spec; + prep_hash = ""; + result = Success; + package = prep; + hash = ""; + })) File "src/pipelines/docs.ml", line 1, characters 0-0: diff --git a/_build/default/src/pipelines/docs.ml b/_build/default/src/pipelines/.formatted/docs.ml index afd53e5..bcd14e3 100644 --- a/_build/default/src/pipelines/docs.ml +++ b/_build/default/src/pipelines/.formatted/docs.ml @@ -11,7 +11,6 @@ end module OpamPackageNameCurrentMap = CurrentMap (OpamPackage.Name.Map) module OpamPackageVersionCurrentMap = CurrentMap (OpamPackage.Version.Map) - let status = function | Error (`Msg msg) -> Monitor.Err msg | Error (`Active _) -> Active @@ -51,8 +50,7 @@ let rec summarise description arr = function items |> current_list_flatten - -let count_refs (all:Package.Set.t) = +let count_refs (all : Package.Set.t) = let prep_jobs : int ref Package.Map.t ref = ref Package.Map.empty in let rec get_prep_job package = @@ -60,168 +58,169 @@ let count_refs (all:Package.Set.t) = let ref = Package.Map.find package !prep_jobs in incr ref with Not_found -> - let dependencies = - Package.universe package |> Package.Universe.deps - in - let () = - List.iter - (fun p -> - get_prep_job p) - dependencies - in + let dependencies = Package.universe package |> Package.Universe.deps in + let () = List.iter (fun p -> get_prep_job p) dependencies in prep_jobs := Package.Map.add package (ref 1) !prep_jobs in Package.Set.iter (fun x -> get_prep_job x) all; !prep_jobs -type 'a compile_job = - { job: 'a Current.t; - monitor : Monitor.pipeline_tree; - } - +type 'a compile_job = { job : 'a Current.t; monitor : Monitor.pipeline_tree } + let compile ~generation ~config ~(blessed : Package.Blessing.Set.t Current.t OpamPackage.Map.t) - (preps : (Prep.t compile_job Package.Map.t)) = - let compilation_jobs : Compile.t compile_job option Package.Map.t ref = ref Package.Map.empty in - let link_jobs : Compile.t compile_job option Package.Map.t ref = ref Package.Map.empty in + (preps : Prep.t compile_job Package.Map.t) = + let compilation_jobs : Compile.t compile_job option Package.Map.t ref = + ref Package.Map.empty + in + let link_jobs : Compile.t compile_job option Package.Map.t ref = + ref Package.Map.empty + in let rec get_compilation_job link package = let name = package |> Package.opam |> OpamPackage.to_string in let extra_deps = Package.universe package |> Package.Universe.extra_link_deps - in + in let job_cache = if link then link_jobs else compilation_jobs in try Package.Map.find package !job_cache - with Not_found -> + with Not_found -> ( let job = Package.Map.find_opt package preps - |> Option.map @@ fun { job=prep; _ } -> + |> Option.map @@ fun { job = prep; _ } -> let dependencies = Package.universe package |> Package.Universe.deps in - begin - if List.length extra_deps > 0 then - Logs.info (fun m -> m "Extra link deps for %s: %s (link: %b)" - name - (extra_deps |> List.map (fun x -> x |> Package.opam |> OpamPackage.to_string) |> String.concat ", ") link) - end; + if List.length extra_deps > 0 then + Logs.info (fun m -> + m "Extra link deps for %s: %s (link: %b)" name + (extra_deps + |> List.map (fun x -> + x |> Package.opam |> OpamPackage.to_string) + |> String.concat ", ") + link); let compile_dependencies_names = List.filter_map (fun p -> - get_compilation_job false p |> Option.map (fun { job; _ } -> (p, job))) + get_compilation_job false p + |> Option.map (fun { job; _ } -> (p, job))) dependencies in let link_dependencies_names = - if link then - List.filter_map - (fun p -> - get_compilation_job false p |> Option.map (fun { job; _ } -> (p, job))) - extra_deps - else - [] - in + if link then + List.filter_map + (fun p -> + get_compilation_job false p + |> Option.map (fun { job; _ } -> (p, job))) + extra_deps + else [] + in let compile_dependencies = compile_dependencies_names |> List.map snd in - let link_dependencies = - link_dependencies_names |> List.map snd - in + let link_dependencies = link_dependencies_names |> List.map snd in let blessing = OpamPackage.Map.find (Package.opam package) blessed |> Current.map (fun b -> Package.Blessing.Set.get b package) in - let deps, jobty = match List.length extra_deps, link with - | 0, _ -> (Current.list_seq compile_dependencies), Compile.CompileAndLink + let deps, jobty = + match (List.length extra_deps, link) with + | 0, _ -> + (Current.list_seq compile_dependencies, Compile.CompileAndLink) | _, false -> - Logs.info (fun m -> m "Creating CompileOnly job"); - (Current.list_seq compile_dependencies), Compile.CompileOnly + Logs.info (fun m -> m "Creating CompileOnly job"); + (Current.list_seq compile_dependencies, Compile.CompileOnly) | _, true -> - Logs.info (fun m -> m "Creating LinkOnly job"); - (Current.list_seq (compile_dependencies @ link_dependencies)), Compile.LinkOnly + Logs.info (fun m -> m "Creating LinkOnly job"); + ( Current.list_seq (compile_dependencies @ link_dependencies), + Compile.LinkOnly ) in let node = - Compile.v ~generation ~config ~name ~blessing - ~deps - ~jobty - prep + Compile.v ~generation ~config ~name ~blessing ~deps ~jobty prep in let monitor = - Monitor.( - Seq - [ - ( "do-deps", - Item prep); - - ("do-compile", Item node); - ]) + Monitor.(Seq [ ("do-deps", Item prep); ("do-compile", Item node) ]) in - (jobty, {job=node; monitor; }) + (jobty, { job = node; monitor }) in match job with | None -> None | Some (CompileAndLink, job) -> - compilation_jobs := Package.Map.add package (Some job) !compilation_jobs; - link_jobs := Package.Map.add package (Some job) !link_jobs; - Some job + compilation_jobs := + Package.Map.add package (Some job) !compilation_jobs; + link_jobs := Package.Map.add package (Some job) !link_jobs; + Some job | Some (CompileOnly, job) -> - compilation_jobs := Package.Map.add package (Some job) !compilation_jobs; - Some job + compilation_jobs := + Package.Map.add package (Some job) !compilation_jobs; + Some job | Some (LinkOnly, job) -> - link_jobs := Package.Map.add package (Some job) !link_jobs; - Some job + link_jobs := Package.Map.add package (Some job) !link_jobs; + Some job) in - let get_compilation_node package _ = - get_compilation_job true package + let get_compilation_node package _ = get_compilation_job true package in + let compile_jobs = + Package.Map.filter_map get_compilation_node preps |> Package.Map.bindings in - let compile_jobs = Package.Map.filter_map get_compilation_node preps |> Package.Map.bindings in compile_jobs -let prep ~config - ~opamfiles (all:Package.Set.t) = - let prep_jobs : Prep.t compile_job Package.Map.t ref = ref Package.Map.empty in +let prep ~config ~opamfiles (all : Package.Set.t) = + let prep_jobs : Prep.t compile_job Package.Map.t ref = + ref Package.Map.empty + in let rec get_prep_job package = try Package.Map.find package !prep_jobs with Not_found -> let job = - let dependencies = - Package.universe package |> Package.Universe.deps + let dependencies = Package.universe package |> Package.Universe.deps in + let dep_opams = + List.map Package.opam dependencies |> OpamPackage.Set.of_list + in + let opams = + OpamPackage.Set.add (Package.opam package) dep_opams + |> OpamPackage.Set.to_list in - let dep_opams = List.map Package.opam dependencies |> OpamPackage.Set.of_list in - let opams = OpamPackage.Set.add (Package.opam package) dep_opams |> OpamPackage.Set.to_list in let ocaml_version = Package.ocaml_version package in let all_opams = Prep.add_base ocaml_version opams in - + let prep_dependencies_names = List.map - (fun p -> - get_prep_job p |> (fun {job; _} -> (p, job))) + (fun p -> get_prep_job p |> fun { job; _ } -> (p, job)) dependencies in - let prep_dependencies = - prep_dependencies_names |> List.map snd - in + let prep_dependencies = prep_dependencies_names |> List.map snd in let base_image = Misc.get_base_image package in - let opamfiles = Current.map (fun x -> - List.filter_map (fun p -> try Some (p, OpamPackage.Map.find p x) with _ -> None) all_opams |> OpamPackage.Map.of_list) opamfiles in + let opamfiles = + Current.map + (fun x -> + List.filter_map + (fun p -> + try Some (p, OpamPackage.Map.find p x) with _ -> None) + all_opams + |> OpamPackage.Map.of_list) + opamfiles + in let node = Prep.v ~config - ~deps:(Current.list_seq prep_dependencies) - ~spec:base_image ~opamfiles ~prep:package + ~deps:(Current.list_seq prep_dependencies) + ~spec:base_image ~opamfiles ~prep:package in let monitor = Monitor.( Seq [ ( "prep", - And (("prep "^Package.id package, Item node) - :: List.map (fun (pkg, compile) -> - ("prep dependency "^Package.id pkg, Item compile)) prep_dependencies_names)); - ] - ) + And + (("prep " ^ Package.id package, Item node) + :: List.map + (fun (pkg, compile) -> + ("prep dependency " ^ Package.id pkg, Item compile)) + prep_dependencies_names) ); + ]) in - { job=node; monitor; } + { job = node; monitor } in prep_jobs := Package.Map.add package job !prep_jobs; @@ -230,24 +229,24 @@ let prep ~config let get_prep_node package = get_prep_job package (* |> Option.map @@ fun (compile, monitor) -> - let node = - Html.v ~generation ~config - ~name:(package |> Package.opam |> OpamPackage.to_string) - ~voodoo:voodoo_gen compile - in - let monitor = - match monitor with - | Monitor.Seq lst -> Monitor.Seq (lst @ [ ("do-html", Item node) ]) - | _ -> assert false - in - let package_status = Monitor.pipeline_state monitor in - let _index = - (* let+ step_list = summarise "" [] monitor *) - (* DEBUGGING THE MEMORY BLOAT -- Skip the summarising for now *) - let+ pipeline_id in - Index.record package pipeline_id package_status [] - in - (node, monitor) *) + let node = + Html.v ~generation ~config + ~name:(package |> Package.opam |> OpamPackage.to_string) + ~voodoo:voodoo_gen compile + in + let monitor = + match monitor with + | Monitor.Seq lst -> Monitor.Seq (lst @ [ ("do-html", Item node) ]) + | _ -> assert false + in + let package_status = Monitor.pipeline_state monitor in + let _index = + (* let+ step_list = summarise "" [] monitor *) + (* DEBUGGING THE MEMORY BLOAT -- Skip the summarising for now *) + let+ pipeline_id in + Index.record package pipeline_id package_status [] + in + (node, monitor) *) in Package.Set.iter (fun x -> ignore (get_prep_node x)) all; !prep_jobs @@ -338,9 +337,7 @@ let v ~config ~opam ~monitor ~migrations () = | Some path -> Index.migrate path | None -> Current.return () in - let generation = - Epoch.v config - in + let generation = Epoch.v config in (* 0) Housekeeping - run migrations *) let* _ = migrations in Log.info (fun f -> f "0) Migrations"); @@ -359,7 +356,9 @@ let v ~config ~opam ~monitor ~migrations () = Log.info (fun f -> f "2) Solver result"); (* 3.a) From solver results, obtain a list of package.version.universe corresponding to prep jobs *) let all_packages_jobs = - solver_result |> Solver.keys |> List.filter_map (fun x -> try Some (Solver.get x) with _ -> None) + solver_result + |> Solver.keys + |> List.filter_map (fun x -> try Some (Solver.get x) with _ -> None) in Log.info (fun f -> f "2.5) Solver result..."); (* 3.b) Expand that list to all the obtainable package.version.universe *) @@ -368,14 +367,19 @@ let v ~config ~opam ~monitor ~migrations () = all_packages_jobs |> List.rev_map Package.all_deps |> List.flatten - |> List.filter (fun pkg -> Ocaml_version.compare (Package.ocaml_version pkg) (Ocaml_version.Releases.v4_04_2) >= 0) + |> List.filter (fun pkg -> + Ocaml_version.compare + (Package.ocaml_version pkg) + Ocaml_version.Releases.v4_04_2 + >= 0) |> Package.Set.of_list in - Log.info (fun f -> f "3) All packages (%d)" (Package.Set.cardinal all_packages)); + Log.info (fun f -> + f "3) All packages (%d)" (Package.Set.cardinal all_packages)); + (* 4) Schedule a somewhat small set of jobs to obtain at least one universe for each package.version *) (* 4a) Decide on a docker tag for each job *) (* 5) Run the preparation step *) - let weekly = Current_cache.Schedule.v ~valid_for:(Duration.of_day 7) () in let repo_opam = @@ -383,24 +387,37 @@ let v ~config ~opam ~monitor ~migrations () = "https://github.com/ocaml/opam-repository.git" in - let opamfiles = - Current.component "opamfiles" |> + let opamfiles = + Current.component "opamfiles" + |> let> repo_opam in - let packages = Package.Set.to_list all_packages |> List.map Package.opam |> OpamPackage.Set.of_list in - let extra = [ - "conf-graphviz.0.1"; - "conf-which.1"; - "base-threads.base"; - "ocaml-options-vanilla.1"; - "base-bigarray.base"; - "base-domains.base"; - "base-nnp.base"; - "base-effects.base"; - "host-arch-x86_64.1"; - "host-system-other.1"; - ] in - let packages = OpamPackage.Set.union (List.map OpamPackage.of_string extra |> OpamPackage.Set.of_list) packages |> OpamPackage.Set.to_list in - Prep.OpamFilesCache.get No_context Prep.OpamFiles.Key.{ repo = repo_opam; packages } + let packages = + Package.Set.to_list all_packages + |> List.map Package.opam + |> OpamPackage.Set.of_list + in + let extra = + [ + "conf-graphviz.0.1"; + "conf-which.1"; + "base-threads.base"; + "ocaml-options-vanilla.1"; + "base-bigarray.base"; + "base-domains.base"; + "base-nnp.base"; + "base-effects.base"; + "host-arch-x86_64.1"; + "host-system-other.1"; + ] + in + let packages = + OpamPackage.Set.union + (List.map OpamPackage.of_string extra |> OpamPackage.Set.of_list) + packages + |> OpamPackage.Set.to_list + in + Prep.OpamFilesCache.get No_context + Prep.OpamFiles.Key.{ repo = repo_opam; packages } in let counts = count_refs all_packages in @@ -412,45 +429,46 @@ let v ~config ~opam ~monitor ~migrations () = let counts = List.sort (fun (_, x) (_, y) -> compare !x !y) counts in - let (_, hd),tl = List.hd counts, List.tl counts in - let (last_count, last_val, count) = List.fold_left (fun (cur_count,cur_val,all) (_, count) -> - if cur_val = !count - then (cur_count + 1, cur_val, all) - else (1, !count, (cur_val, cur_count) :: all)) (1, !hd, []) tl in + let (_, hd), tl = (List.hd counts, List.tl counts) in + let last_count, last_val, count = + List.fold_left + (fun (cur_count, cur_val, all) (_, count) -> + if cur_val = !count then (cur_count + 1, cur_val, all) + else (1, !count, (cur_val, cur_count) :: all)) + (1, !hd, []) tl + in let all_counts = (last_val, last_count) :: count in - List.iter (fun (v, count) -> - Printf.printf "%d %d\n%!" v count) all_counts; - - let cache_packages = List.fold_left (fun acc (v, count) -> - if !count > Config.cache_threshold config - then Package.Set.add v acc - else acc) Package.Set.empty counts + List.iter (fun (v, count) -> Printf.printf "%d %d\n%!" v count) all_counts; + + let cache_packages = + List.fold_left + (fun acc (v, count) -> + if !count > Config.cache_threshold config then Package.Set.add v acc + else acc) + Package.Set.empty counts in Package.add_important_packages cache_packages; Log.info (fun f -> - f "4) Cache packages (%d)" (Package.Set.cardinal cache_packages)); + f "4) Cache packages (%d)" (Package.Set.cardinal cache_packages)); (* let cumulative = - let x = ref 0 in - List.map (fun (v, count) -> - let y = !x + count in - x := y; - (v, y)) all_counts in + let x = ref 0 in + List.map (fun (v, count) -> + let y = !x + count in + x := y; + (v, y)) all_counts in - List.iter (fun (v, count) -> - Printf.printf "%d %d\n%!" v count) cumulative; - - ignore @@ exit 0; *) + List.iter (fun (v, count) -> + Printf.printf "%d %d\n%!" v count) cumulative; + ignore @@ exit 0; *) let prepped' : Prep.t compile_job Package.Map.t = - prep ~config - ~opamfiles all_packages + prep ~config ~opamfiles all_packages in - Log.info (fun f -> - f ".. %d prepped nodes" (Package.Map.cardinal prepped')); + Log.info (fun f -> f ".. %d prepped nodes" (Package.Map.cardinal prepped')); Log.info (fun f -> f "5) Prep nodes"); (* 6) Promote packages to the main tree *) @@ -470,7 +488,7 @@ let v ~config ~opam ~monitor ~migrations () = in let by_opam_package = Package.Map.fold - (fun package {job=prep; _ } opam_map -> + (fun package { job = prep; _ } opam_map -> let opam = Package.opam package in let job = let+ job = Current.state ~hidden:true prep in @@ -497,29 +515,32 @@ let v ~config ~opam ~monitor ~migrations () = (* 7) Odoc compile and html-generate artifacts *) let html, html_input_node, package_pipeline_tree, html2 = - let compile_monitor = - compile ~generation ~config ~blessed - prepped' + let compile_monitor = compile ~generation ~config ~blessed prepped' in + let html2 = + List.map + (fun (pkg, { job = compile; _ }) -> (pkg, compile)) + compile_monitor in - let html2 = List.map (fun (pkg, { job=compile; _ }) -> (pkg, compile)) compile_monitor in Log.info (fun f -> f ".. %d compilation nodes" (List.length compile_monitor)); let c, compile_node = compile_monitor - |> List.map (fun (a, {job=b; _}) -> (a, b)) + |> List.map (fun (a, { job = b; _ }) -> (a, b)) |> compile_hierarchical_collapse ~input:solver_result_c in ( c |> List.to_seq |> Package.Map.of_seq, compile_node, compile_monitor - |> List.map (fun (a, { monitor=b; _}) -> (a, b)) + |> List.map (fun (a, { monitor = b; _ }) -> (a, b)) |> List.to_seq |> Package.Map.of_seq, - html2 |> Package.Map.of_list) + html2 |> Package.Map.of_list ) in Log.info (fun f -> f "7) Odoc compile nodes"); - let prep_pipeline_tree = Package.Map.map (fun (p : Prep.t compile_job) -> p.monitor) prepped' in + let prep_pipeline_tree = + Package.Map.map (fun (p : Prep.t compile_job) -> p.monitor) prepped' + in (* 7.b) Inform the monitor *) let () = dune build @fmt failed "/usr/bin/env" "bash" "-c" "opam exec -- dune build @fmt --ignore-promoted-rules || (echo "dune build @fmt failed"; exit 2)" failed with exit status 2 2025-04-22 12:40.13: Job failed: Failed: Build failed