bpf: notify offload JITs about optimizations

Let offload JITs know when instructions are replaced and optimized
out, so they can update their state appropriately.  The optimizations
are best effort, if JIT returns an error from any callback verifier
will stop notifying it as state may now be out of sync, but the
verifier continues making progress.

Signed-off-by: Jakub Kicinski <[email protected]>
Reviewed-by: Quentin Monnet <[email protected]>
Signed-off-by: Alexei Starovoitov <[email protected]>

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index e734f163bd0b9ae05c199fde7bd4e3dc1bfed542..3851529062ecb46f984715b5271234ee017b2829 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -268,9 +268,15 @@ struct bpf_verifier_ops {
 };
 
 struct bpf_prog_offload_ops {
+       /* verifier basic callbacks */
        int (*insn_hook)(struct bpf_verifier_env *env,
                         int insn_idx, int prev_insn_idx);
        int (*finalize)(struct bpf_verifier_env *env);
+       /* verifier optimization callbacks (called after .finalize) */
+       int (*replace_insn)(struct bpf_verifier_env *env, u32 off,
+                           struct bpf_insn *insn);
+       int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt);
+       /* program management callbacks */
        int (*prepare)(struct bpf_prog *prog);
        int (*translate)(struct bpf_prog *prog);
        void (*destroy)(struct bpf_prog *prog);
@@ -283,6 +289,7 @@ struct bpf_prog_offload {
        void                    *dev_priv;
        struct list_head        offloads;
        bool                    dev_state;
+       bool                    opt_failed;
        void                    *jited_image;
        u32                     jited_len;
 };

diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index f3ae00ee5516e9d176f6f4c8a9faba8a01731952..0620e418dde5d48161d659c291839a0e93f93621 100644 (file)
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -266,5 +266,10 @@ int bpf_prog_offload_verifier_prep(struct bpf_prog *prog);
 int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env,
                                 int insn_idx, int prev_insn_idx);
 int bpf_prog_offload_finalize(struct bpf_verifier_env *env);
+void
+bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off,
+                             struct bpf_insn *insn);
+void
+bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt);
 
 #endif /* _LINUX_BPF_VERIFIER_H */

diff --git a/kernel/bpf/offload.c b/kernel/bpf/offload.c
index 54cf2b9c44a4570e8007848d1b530d367aefe64f..39dba8c9033128d75817307accd7b8d277637fc6 100644 (file)
--- a/kernel/bpf/offload.c
+++ b/kernel/bpf/offload.c
@@ -173,6 +173,41 @@ int bpf_prog_offload_finalize(struct bpf_verifier_env *env)
        return ret;
 }
 
+void
+bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off,
+                             struct bpf_insn *insn)
+{
+       const struct bpf_prog_offload_ops *ops;
+       struct bpf_prog_offload *offload;
+       int ret = -EOPNOTSUPP;
+
+       down_read(&bpf_devs_lock);
+       offload = env->prog->aux->offload;
+       if (offload) {
+               ops = offload->offdev->ops;
+               if (!offload->opt_failed && ops->replace_insn)
+                       ret = ops->replace_insn(env, off, insn);
+               offload->opt_failed |= ret;
+       }
+       up_read(&bpf_devs_lock);
+}
+
+void
+bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
+{
+       struct bpf_prog_offload *offload;
+       int ret = -EOPNOTSUPP;
+
+       down_read(&bpf_devs_lock);
+       offload = env->prog->aux->offload;
+       if (offload) {
+               if (!offload->opt_failed && offload->offdev->ops->remove_insns)
+                       ret = offload->offdev->ops->remove_insns(env, off, cnt);
+               offload->opt_failed |= ret;
+       }
+       up_read(&bpf_devs_lock);
+}
+
 static void __bpf_prog_offload_destroy(struct bpf_prog *prog)
 {
        struct bpf_prog_offload *offload = prog->aux->offload;

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index f2c49b4235df8d6bdb08ece70e27484cc501f055..8cfe39ef770f4c5447f62f6d691bfbd3eb6a3feb 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -6558,6 +6558,9 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
        unsigned int orig_prog_len = env->prog->len;
        int err;
 
+       if (bpf_prog_is_dev_bound(env->prog->aux))
+               bpf_prog_offload_remove_insns(env, off, cnt);
+
        err = bpf_remove_insns(env->prog, off, cnt);
        if (err)
                return err;
@@ -6632,6 +6635,9 @@ static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)
                else
                        continue;
 
+               if (bpf_prog_is_dev_bound(env->prog->aux))
+                       bpf_prog_offload_replace_insn(env, i, &ja);
+
                memcpy(insn, &ja, sizeof(ja));
        }
 }