diff --git a/sound/soc/sh/rcar/adg.c b/sound/soc/sh/rcar/adg.c
index 85a33ac0a5c4..9665c1fa7216 100644
--- a/sound/soc/sh/rcar/adg.c
+++ b/sound/soc/sh/rcar/adg.c
@@ -537,16 +537,14 @@ static void rsnd_adg_get_clkout(struct rsnd_priv *priv,
 			clk = clk_register_fixed_rate(dev, clkout_name[i],
 						      parent_clk_name, 0,
 						      req_rate);
-			if (!IS_ERR(clk)) {
-				adg->onecell.clks	= adg->clkout;
-				adg->onecell.clk_num	= CLKOUTMAX;
-
+			adg->clkout[i] = ERR_PTR(-ENOENT);
+			if (!IS_ERR(clk))
 				adg->clkout[i] = clk;
-
-				of_clk_add_provider(np, of_clk_src_onecell_get,
-						    &adg->onecell);
-			}
 		}
+		adg->onecell.clks	= adg->clkout;
+		adg->onecell.clk_num	= CLKOUTMAX;
+		of_clk_add_provider(np, of_clk_src_onecell_get,
+				    &adg->onecell);
 	}
 
 	adg->ckr = ckr;
@@ -589,5 +587,10 @@ int rsnd_adg_probe(struct rsnd_priv *priv)
 
 void rsnd_adg_remove(struct rsnd_priv *priv)
 {
+	struct device *dev = rsnd_priv_to_dev(priv);
+	struct device_node *np = dev->of_node;
+
+	of_clk_del_provider(np);
+
 	rsnd_adg_clk_disable(priv);
 }
diff --git a/sound/soc/sh/rcar/src.c b/sound/soc/sh/rcar/src.c
index 42db48db09ba..20b5b2ec625e 100644
--- a/sound/soc/sh/rcar/src.c
+++ b/sound/soc/sh/rcar/src.c
@@ -167,6 +167,7 @@ static int rsnd_src_hw_params(struct rsnd_mod *mod,
 	 *	dpcm_fe_dai_hw_params()
 	 *	dpcm_be_dai_hw_params()
 	 */
+	src->convert_rate = 0;
 	if (fe->dai_link->dynamic) {
 		int stream = substream->stream;
 		struct snd_soc_dpcm *dpcm;
@@ -414,8 +415,6 @@ static int rsnd_src_quit(struct rsnd_mod *mod,
 
 	rsnd_mod_power_off(mod);
 
-	src->convert_rate = 0;
-
 	/* reset sync convert_rate */
 	src->sync.val = 0;