pull: Add options to slow down plls

We need to expose more control over how fast we write to disk
in order to allow OS vendors to more finely tune latency vs
throughput for OS updates.

See openshift/machine-config-operator#1897

This `OSTREE_PULL_MAX_OUTSTANDING_WRITES` is a crude hack
I'm using for testing; need to also expose this via API, document it etc.

Second, add a `per-object-fsync` option which goes back to invoking
`fsync()` on each object just after writing it.

Combined, these two ensure natural "backpressure" against trying
to fsync a huge amount of data all at once, which I believe is
what is leading to the huge etcd fsync latency.

src/libostree/ostree-repo-commit.c

@@ -420,7 +420,7 @@ commit_loose_regfile_object (OstreeRepo        *self,
   /* Ensure that in case of a power cut, these files have the data we
    * want.   See http://lwn.net/Articles/322823/
    */
-  if (!self->in_transaction && !self->disable_fsync)
+  if (!self->disable_fsync && (self->per_object_fsync || !self->in_transaction))
     {
       if (fsync (tmpf->fd) == -1)
         return glnx_throw_errno_prefix (error, "fsync");

src/libostree/ostree-repo-private.h

@@ -147,6 +147,7 @@ struct OstreeRepo {
   GError *writable_error;
   gboolean in_transaction;
   gboolean disable_fsync;
+  gboolean per_object_fsync;
   gboolean disable_xattrs;
   guint zlib_compression_level;
   GHashTable *loose_object_devino_hash;

src/libostree/ostree-repo-pull-private.h

@@ -91,6 +91,7 @@ typedef struct {
   GHashTable       *pending_fetch_content; /* Map<checksum,FetchObjectData> */
   GHashTable       *pending_fetch_delta_superblocks; /* Set<FetchDeltaSuperData> */
   GHashTable       *pending_fetch_deltaparts; /* Set<FetchStaticDeltaData> */
+  guint             max_outstanding_write_requests;
   guint             n_outstanding_metadata_fetches;
   guint             n_outstanding_metadata_write_requests;
   guint             n_outstanding_content_fetches;

src/libostree/ostree-repo-pull.c

@@ -380,7 +380,7 @@ fetcher_queue_is_full (OtPullData *pull_data)
       ((pull_data->n_outstanding_metadata_write_requests +
         pull_data->n_outstanding_content_write_requests +
         pull_data->n_outstanding_deltapart_write_requests) >=
-         _OSTREE_MAX_OUTSTANDING_WRITE_REQUESTS);
+         pull_data->max_outstanding_write_requests);
   return fetch_full || deltas_full || writes_full;
 }
 
@@ -3456,6 +3456,14 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   if (!opt_n_network_retries_set)
     pull_data->n_network_retries = DEFAULT_N_NETWORK_RETRIES;
 
+  {
+    const char *writes = getenv ("OSTREE_PULL_MAX_OUTSTANDING_WRITES");
+    if (writes != NULL)
+      pull_data->max_outstanding_write_requests = g_ascii_strtoull (writes, NULL, 10);
+    else
+      pull_data->max_outstanding_write_requests = _OSTREE_MAX_OUTSTANDING_WRITE_REQUESTS;
+  }
+
   pull_data->repo = self;
   pull_data->progress = progress;
 

src/libostree/ostree-repo.c

@@ -2922,6 +2922,10 @@ reload_core_config (OstreeRepo          *self,
       ostree_repo_set_disable_fsync (self, TRUE);
   }
 
+  if (!ot_keyfile_get_boolean_with_default (self->config, "core", "per-object-fsync",
+                                            FALSE, &self->per_object_fsync, error))
+    return FALSE;
+
   /* See https://github.com/ostreedev/ostree/issues/758 */
   if (!ot_keyfile_get_boolean_with_default (self->config, "core", "disable-xattrs",
                                             FALSE, &self->disable_xattrs, error))