Speedup opam update when nothing changed

master_changes.md

@@ -95,6 +95,7 @@ users)
 ## Shell
 
 ## Internal
+  * Ensure each repositories stored in repos-config is associated with an URL [#6249 @kit-ty-kate]
 
 ## Internal: Windows
 
@@ -125,5 +126,6 @@ users)
 ## opam-solver
 
 ## opam-format
+  * `OpamFile.Repos_config.t`: change the type to not allow repositories without an URL [#6249 @kit-ty-kate]
 
 ## opam-core

src/client/opamAdminCommand.ml

@@ -882,6 +882,8 @@ let get_virtual_switch_state repo_root env =
     repo_name = OpamRepositoryName.of_string "local";
     repo_url = OpamUrl.empty;
     repo_trust = None;
+    repo_etag = None;
+    repo_last_modified = None;
   } in
   let repo_file = OpamRepositoryPath.repo repo_root in
   let repo_def = OpamFile.Repo.safe_read repo_file in

src/client/opamAdminRepoUpgrade.ml

@@ -196,7 +196,7 @@ let do_upgrade repo_root =
                 let base = OpamFilename.Base.of_string "package.patch" in
                 OpamFilename.create dir base
               in
-              OpamDownload.download_as ~overwrite:false url f @@| fun () ->
+              OpamDownload.download_as ~etag:None ~last_modified:None ~overwrite:false url f @@| fun _was_downloaded ->
               let hash = OpamHash.compute (OpamFilename.to_string f) in
               Hashtbl.add url_md5 url hash;
               Some hash)),

src/client/opamClient.ml

@@ -1836,7 +1836,7 @@ let init
       try
         (* Create the content of ~/.opam/config *)
         let repos = match repo with
-          | Some r -> [r.repo_name, (r.repo_url, r.repo_trust)]
+          | Some r -> [r.repo_name, (r.repo_url, r.repo_trust, r.repo_etag, r.repo_last_modified)]
           | None -> OpamFile.InitConfig.repositories init_config
         in
         let config =
@@ -1885,10 +1885,7 @@ let init
           else config
         in
         OpamFile.Config.write config_f config;
-        let repos_config =
-          OpamRepositoryName.Map.of_list repos |>
-          OpamRepositoryName.Map.map OpamStd.Option.some
-        in
+        let repos_config = OpamRepositoryName.Map.of_list repos in
         OpamFile.Repos_config.write (OpamPath.repos_config root)
           repos_config;
 

src/client/opamCommands.ml

@@ -135,7 +135,7 @@ let get_init_config ~no_sandboxing ~no_default_config_file ~add_config_file =
         | Some f -> OpamFile.make f
         | None ->
           let f = OpamFilename.of_string (OpamSystem.temp_file "conf") in
-          OpamProcess.Job.run (OpamDownload.download_as ~overwrite:false url f);
+          let _was_downloaded = OpamProcess.Job.run (OpamDownload.download_as ~etag:None ~last_modified:None ~overwrite:false url f) in
           let hash = OpamHash.compute ~kind:`SHA256 (OpamFilename.to_string f) in
           if OpamConsole.confirm
               "Using configuration file from %s. \
@@ -478,7 +478,7 @@ let init cli =
     let repo =
       OpamStd.Option.map (fun url ->
           let repo_url = OpamUrl.parse ?backend:repo_kind ~from_file:false url in
-          { repo_name; repo_url; repo_trust = None })
+          { repo_name; repo_url; repo_trust = None; repo_etag = None; repo_last_modified = None })
         repo_url
     in
     let gt, rt, default_compiler =

src/client/opamInitDefaults.ml

@@ -168,7 +168,7 @@ let (@|) g f = OpamStd.Op.(g @* f) ()
 let init_config ?(sandboxing=true) () =
   I.empty |>
   I.with_repositories
-    [OpamRepositoryName.of_string "default", (repository_url, None)] |>
+    [OpamRepositoryName.of_string "default", (repository_url, None, None, None)] |>
   I.with_default_compiler default_compiler |>
   I.with_default_invariant default_invariant |>
   I.with_eval_variables eval_variables |>

src/client/opamRepositoryCommand.ml

@@ -80,7 +80,8 @@ let add rt name url trust_anchors =
       (OpamUrl.to_string url)
   | None ->
     let repo = { repo_name = name; repo_url = url;
-                 repo_trust = trust_anchors; }
+                 repo_trust = trust_anchors;
+                 repo_etag = None; repo_last_modified = None; }
     in
     if OpamFilename.exists_dir (OpamRepositoryPath.root root name) ||
        OpamFilename.exists (OpamRepositoryPath.tar root name)

src/format/opamFile.ml

@@ -1706,7 +1706,7 @@ module InitConfigSyntax = struct
 
   type t = {
     opam_version : opam_version;
-    repositories : (repository_name * (url * trust_anchors option)) list;
+    repositories : (repository_name * (url * trust_anchors option * string option * string option)) list;
     default_compiler : formula;
     default_invariant : formula;
     jobs : int option;
@@ -1795,22 +1795,26 @@ module InitConfigSyntax = struct
   }
 
   let pp_repository_def =
-    Pp.V.map_options_3
+    Pp.V.map_options_5
       (Pp.V.string -|
        Pp.of_module "repository" (module OpamRepositoryName))
-      (Pp.opt @@ Pp.singleton -| Pp.V.url)
+      (Pp.singleton -| Pp.V.url)
       (Pp.map_list Pp.V.string)
       (Pp.opt @@
        Pp.singleton -| Pp.V.int -|
-       OpamPp.check ~name:"quorum" ~errmsg:"quorum must be >= 0" ((<=) 0)) -|
+       OpamPp.check ~name:"quorum" ~errmsg:"quorum must be >= 0" ((<=) 0))
+      (Pp.opt (Pp.singleton -| Pp.V.string))
+      (Pp.opt (Pp.singleton -| Pp.V.string)) -|
     Pp.pp
-      (fun ~pos:_ (name, url, fingerprints, quorum) ->
-         name, url,
-         match fingerprints with [] -> None | fingerprints ->
-           Some {fingerprints; quorum = OpamStd.Option.default 1 quorum})
-      (fun (name, url, ta) -> match ta with
-         | Some ta ->  name, url, ta.fingerprints, Some ta.quorum
-         | None -> name, url, [], None)
+      (fun ~pos:_ (name, url, fingerprints, quorum, etag, last_modified) ->
+         let ta =
+           match fingerprints with [] -> None | fingerprints ->
+             Some {fingerprints; quorum = OpamStd.Option.default 1 quorum}
+         in
+         (name, url, ta, etag, last_modified))
+      (fun (name, url, ta, etag, last_modified) -> match ta with
+         | Some ta -> (name, url, ta.fingerprints, Some ta.quorum, etag, last_modified)
+         | None -> (name, url, [], None, etag, last_modified))
 
   let fields =
     [
@@ -1821,10 +1825,8 @@ module InitConfigSyntax = struct
         with_repositories repositories
         (Pp.V.map_list ~depth:1 @@
          pp_repository_def -|
-         Pp.pp (fun ~pos -> function
-             | (name, Some url, ta) -> (name, (url, ta))
-             | (_, None, _) -> Pp.bad_format ~pos "Missing repository URL")
-           (fun (name, (url, ta)) -> (name, Some url, ta)));
+         Pp.pp (fun ~pos:_ (name, url, ta, etag, last_modified) -> (name, (url, ta, etag, last_modified)))
+           (fun (name, (url, ta, etag, last_modified)) -> (name, url, ta, etag, last_modified)));
       "default-compiler", Pp.ppacc
         with_default_compiler default_compiler
         (Pp.V.package_formula `Disj Pp.V.(constraints Pp.V.version));
@@ -1965,7 +1967,7 @@ module Repos_configSyntax = struct
   let format_version = OpamVersion.of_string "2.0"
   let file_format_version = OpamVersion.of_string "2.0"
 
-  type t = ((url * trust_anchors option) option) OpamRepositoryName.Map.t
+  type t = (url * trust_anchors option * string option * string option) OpamRepositoryName.Map.t
 
   let empty = OpamRepositoryName.Map.empty
 
@@ -1975,12 +1977,8 @@ module Repos_configSyntax = struct
       ((Pp.V.map_list ~depth:1 @@
         InitConfigSyntax.pp_repository_def -|
         Pp.pp
-          (fun ~pos:_ -> function
-             | (name, Some url, ta) -> name, Some (url, ta)
-             | (name, None, _) -> name, None)
-          (fun (name, def) -> match def with
-             | Some (url, ta) -> name, Some url, ta
-             | None -> name, None, None)) -|
+          (fun ~pos:_ (name, url, ta, etag, last_modified) -> (name, (url, ta, etag, last_modified)))
+          (fun (name, (url, ta, etag, last_modified)) -> (name, url, ta, etag, last_modified))) -|
        Pp.of_pair "repository-url-list"
          OpamRepositoryName.Map.(of_list, bindings));
   ]

src/format/opamFile.mli

@@ -254,7 +254,7 @@ module InitConfig: sig
   include IO_FILE
 
   val opam_version: t -> opam_version
-  val repositories: t -> (repository_name * (url * trust_anchors option)) list
+  val repositories: t -> (repository_name * (url * trust_anchors option * string option * string option)) list
   val default_compiler: t -> formula
   val default_invariant: t -> formula
   val jobs: t -> int option
@@ -274,7 +274,7 @@ module InitConfig: sig
 
   val with_opam_version: opam_version -> t -> t
   val with_repositories:
-    (repository_name * (url * trust_anchors option)) list -> t -> t
+    (repository_name * (url * trust_anchors option * string option * string option)) list -> t -> t
   val with_default_compiler: formula -> t -> t
   val with_default_invariant: formula -> t -> t
   val with_jobs: int option -> t -> t
@@ -1028,7 +1028,7 @@ module Repo_config_legacy : sig
 end
 
 module Repos_config: sig
-  type t = (url * trust_anchors option) option OpamRepositoryName.Map.t
+  type t = (url * trust_anchors option * string option * string option) OpamRepositoryName.Map.t
   include IO_FILE with type t := t
   module BestEffort: BestEffortRead with type t := t
 end

src/format/opamFormat.ml

@@ -244,6 +244,20 @@ module V = struct
       pp4 -|
     pp (fun ~pos:_ (((a,b),c),d) -> a,b,c,d) (fun (a,b,c,d) -> ((a,b),c),d)
 
+  let map_options_5 pp1 pp2 pp3 pp4 pp5 pp6 =
+    option_depth 5 -|
+    option_strict -| map_option_contents
+      (option_strict -| map_option_contents
+         (option_strict -| map_option_contents
+            (option_strict -| map_option_contents
+               (option_strict -| map_option_contents
+                  pp1 pp2)
+               pp3)
+            pp4)
+         pp5)
+      pp6 -|
+    pp (fun ~pos:_ (((((a,b),c),d),e),f) -> a,b,c,d,e,f) (fun (a,b,c,d,e,f) -> ((((a,b),c),d),e),f)
+
   let map_pair pp1 pp2 =
     pp ~name:(Printf.sprintf "[%s %s]" pp1.ppname pp2.ppname)
       (fun ~pos:_ v -> match v.pelem with

src/format/opamFormat.mli

@@ -101,6 +101,13 @@ module V : sig
     (value list, 'b) t -> (value list, 'c) t -> (value list, 'd) t ->
     (value, 'a * 'b * 'c * 'd) t
 
+  (** Maps over five options (e.g. [v {op1} {op2} {op3} {op4} {op5}]) *)
+  val map_options_5 :
+    (value, 'a) t ->
+    (value list, 'b) t -> (value list, 'c) t -> (value list, 'd) t ->
+    (value list, 'e) t -> (value list, 'f) t ->
+    (value, 'a * 'b * 'c * 'd * 'e * 'f) t
+
   (** A pair is simply a list with two elements in the [value] type *)
   val map_pair :
     (value, 'a) t ->

src/format/opamTypes.mli

@@ -176,9 +176,11 @@ type trust_anchors = {
 
 (** Repositories *)
 type repository = {
-  repo_name    : repository_name;
-  repo_url     : url;
-  repo_trust   : trust_anchors option;
+  repo_name          : repository_name;
+  repo_url           : url;
+  repo_trust         : trust_anchors option;
+  repo_etag          : string option;
+  repo_last_modified : string option;
 }
 
 (** {2 Variable-based filters} *)