drm/nv50/pm: introduce hwsq-based memory reclocking

More work needs to be done on supporting the different memory types.

v2 (Ben Skeggs):
- fixed up conflicts from not having pausing patch first
- restructured code somewhat to fit with how all the other code works
- fixed bug where incorrect mpll_ctrl could get set sometimes
- removed stuff that's cargo-culted from the binary driver
- merged nv92+ display disable into hwsq
- fixed incorrect opcode 0x5f magic at end of ucode

Signed-off-by: Martin Peres <martin.peres@ensi-bourges.fr>
Signed-off-by: Ben Skeggs <bskeggs@redhat.com>

drivers/gpu/drm/nouveau/nouveau_hwsq.h

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#ifndef __NOUVEAU_HWSQ_H__
+#define __NOUVEAU_HWSQ_H__
+
+struct hwsq_ucode {
+	u8 data[0x200];
+	union {
+		u8  *u08;
+		u16 *u16;
+		u32 *u32;
+	} ptr;
+	u16 len;
+
+	u32 reg;
+	u32 val;
+};
+
+static inline void
+hwsq_init(struct hwsq_ucode *hwsq)
+{
+	hwsq->ptr.u08 = hwsq->data;
+	hwsq->reg = 0xffffffff;
+	hwsq->val = 0xffffffff;
+}
+
+static inline void
+hwsq_fini(struct hwsq_ucode *hwsq)
+{
+	do {
+		*hwsq->ptr.u08++ = 0x7f;
+		hwsq->len = hwsq->ptr.u08 - hwsq->data;
+	} while (hwsq->len & 3);
+	hwsq->ptr.u08 = hwsq->data;
+}
+
+static inline void
+hwsq_unkn(struct hwsq_ucode *hwsq, u8 v0)
+{
+	*hwsq->ptr.u08++ = v0;
+}
+
+static inline void
+hwsq_op5f(struct hwsq_ucode *hwsq, u8 v0, u8 v1)
+{
+	*hwsq->ptr.u08++ = 0x5f;
+	*hwsq->ptr.u08++ = v0;
+	*hwsq->ptr.u08++ = v1;
+}
+
+static inline void
+hwsq_wr32(struct hwsq_ucode *hwsq, u32 reg, u32 val)
+{
+	if (val != hwsq->val) {
+		if ((val & 0xffff0000) == (hwsq->val & 0xffff0000)) {
+			*hwsq->ptr.u08++ = 0x42;
+			*hwsq->ptr.u16++ = (val & 0x0000ffff);
+		} else {
+			*hwsq->ptr.u08++ = 0xe2;
+			*hwsq->ptr.u32++ = val;
+		}
+
+		hwsq->val = val;
+	}
+
+	if ((reg & 0xffff0000) == (hwsq->reg & 0xffff0000)) {
+		*hwsq->ptr.u08++ = 0x40;
+		*hwsq->ptr.u16++ = (reg & 0x0000ffff);
+	} else {
+		*hwsq->ptr.u08++ = 0xe0;
+		*hwsq->ptr.u32++ = reg;
+	}
+	hwsq->reg = reg;
+}
+
+#endif

drivers/gpu/drm/nouveau/nv50_pm.c

@@ -27,6 +27,7 @@
 #include "nouveau_bios.h"
 #include "nouveau_hw.h"
 #include "nouveau_pm.h"
+#include "nouveau_hwsq.h"
 
 enum clk_src {
 	clk_src_crystal,
@@ -351,6 +352,9 @@ nv50_pm_clocks_get(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 }
 
 struct nv50_pm_state {
+	struct hwsq_ucode mclk_hwsq;
+	u32 mscript;
+
 	u32 emast;
 	u32 nctrl;
 	u32 ncoef;
@@ -359,10 +363,6 @@ struct nv50_pm_state {
 
 	u32 amast;
 	u32 pdivs;
-
-	u32 mscript;
-	u32 mctrl;
-	u32 mcoef;
 };
 
 static u32
@@ -415,6 +415,80 @@ clk_same(u32 a, u32 b)
 	return ((a / 1000) == (b / 1000));
 }
 
+static int
+calc_mclk(struct drm_device *dev, u32 freq, struct hwsq_ucode *hwsq)
+{
+	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct pll_lims pll;
+	u32 mast = nv_rd32(dev, 0x00c040);
+	u32 ctrl = nv_rd32(dev, 0x004008);
+	u32 coef = nv_rd32(dev, 0x00400c);
+	u32 orig = ctrl;
+	u32 crtc_mask = 0;
+	int N, M, P;
+	int ret, i;
+
+	/* use pcie refclock if possible, otherwise use mpll */
+	ctrl &= ~0x81ff0200;
+	if (clk_same(freq, read_clk(dev, clk_src_href))) {
+		ctrl |= 0x00000200 | (pll.log2p_bias << 19);
+	} else {
+		ret = calc_pll(dev, 0x4008, &pll, freq, &N, &M, &P);
+		if (ret == 0)
+			return -EINVAL;
+
+		ctrl |= 0x80000000 | (P << 22) | (P << 16);
+		ctrl |= pll.log2p_bias << 19;
+		coef  = (N << 8) | M;
+	}
+
+	mast &= ~0xc0000000; /* get MCLK_2 from HREF */
+	mast |=  0x0000c000; /* use MCLK_2 as MPLL_BYPASS clock */
+
+	/* determine active crtcs */
+	for (i = 0; i < 2; i++) {
+		if (nv_rd32(dev, NV50_PDISPLAY_CRTC_C(i, CLOCK)))
+			crtc_mask |= (1 << i);
+	}
+
+	/* build the ucode which will reclock the memory for us */
+	hwsq_init(hwsq);
+	if (crtc_mask) {
+		hwsq_op5f(hwsq, crtc_mask, 0x00); /* wait for scanout */
+		hwsq_op5f(hwsq, crtc_mask, 0x01); /* wait for vblank */
+	}
+	if (dev_priv->chipset >= 0x92)
+		hwsq_wr32(hwsq, 0x611200, 0x00003300); /* disable scanout */
+	hwsq_unkn(hwsq, 0xb0); /* disable bus access */
+	hwsq_op5f(hwsq, 0x00, 0x01); /* no idea :s */
+
+	/* prepare memory controller */
+	hwsq_wr32(hwsq, 0x1002d4, 0x00000001); /* precharge banks and idle */
+	hwsq_wr32(hwsq, 0x1002d0, 0x00000001); /* force refresh */
+	hwsq_wr32(hwsq, 0x100210, 0x00000000); /* stop the automatic refresh */
+	hwsq_wr32(hwsq, 0x1002dc, 0x00000001); /* start self refresh mode */
+
+	/* reclock memory */
+	hwsq_wr32(hwsq, 0xc040, mast);
+	hwsq_wr32(hwsq, 0x4008, orig | 0x00000200); /* bypass MPLL */
+	hwsq_wr32(hwsq, 0x400c, coef);
+	hwsq_wr32(hwsq, 0x4008, ctrl);
+
+	/* restart memory controller */
+	hwsq_wr32(hwsq, 0x1002d4, 0x00000001); /* precharge banks and idle */
+	hwsq_wr32(hwsq, 0x1002dc, 0x00000000); /* stop self refresh mode */
+	hwsq_wr32(hwsq, 0x100210, 0x80000000); /* restart automatic refresh */
+	hwsq_unkn(hwsq, 0x07); /* wait for the PLL to stabilize (12us) */
+
+	hwsq_unkn(hwsq, 0x0b); /* may be unnecessary: causes flickering */
+	hwsq_unkn(hwsq, 0xd0); /* enable bus access again */
+	hwsq_op5f(hwsq, 0x00, 0x00); /* no idea, reverse of 0x00, 0x01? */
+	if (dev_priv->chipset >= 0x92)
+		hwsq_wr32(hwsq, 0x611200, 0x00003330); /* enable scanout */
+	hwsq_fini(hwsq);
+	return 0;
+}
+
 void *
 nv50_pm_clocks_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 {
@@ -462,23 +536,16 @@ nv50_pm_clocks_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 		info->scoef  = (N << 8) | M;
 	}
 
-	/* memory: use pcie refclock if possible, otherwise use mpll */
-	info->mscript = perflvl->memscript;
-	if (clk_same(perflvl->memory, read_clk(dev, clk_src_href))) {
-		info->mctrl = 0x00000200 | (pll.log2p_bias << 19);
-		info->mcoef = nv_rd32(dev, 0x400c);
-	} else
+	/* memory: build hwsq ucode which we'll use to reclock memory */
+	info->mclk_hwsq.len = 0;
 	if (perflvl->memory) {
-		clk = calc_pll(dev, 0x4008, &pll, perflvl->memory,
-			       &N, &M, &P1);
-		if (clk == 0)
+		clk = calc_mclk(dev, perflvl->memory, &info->mclk_hwsq);
+		if (clk < 0) {
+			ret = clk;
 			goto error;
+		}
 
-		info->mctrl  = 0x80000000 | (P1 << 22) | (P1 << 16);
-		info->mctrl |= pll.log2p_bias << 19;
-		info->mcoef  = (N << 8) | M;
-	} else {
-		info->mctrl = 0x00000000;
+		info->mscript = perflvl->memscript;
 	}
 
 	/* vdec: avoid modifying xpll until we know exactly how the other
@@ -537,6 +604,44 @@ error:
 	return ERR_PTR(ret);
 }
 
+static int
+prog_mclk(struct drm_device *dev, struct hwsq_ucode *hwsq)
+{
+	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	u32 hwsq_data, hwsq_kick;
+	int i;
+
+	if (dev_priv->chipset < 0x90) {
+		hwsq_data = 0x001400;
+		hwsq_kick = 0x00000003;
+	} else {
+		hwsq_data = 0x080000;
+		hwsq_kick = 0x00000001;
+	}
+
+	/* upload hwsq ucode */
+	nv_mask(dev, 0x001098, 0x00000008, 0x00000000);
+	nv_wr32(dev, 0x001304, 0x00000000);
+	for (i = 0; i < hwsq->len / 4; i++)
+		nv_wr32(dev, hwsq_data + (i * 4), hwsq->ptr.u32[i]);
+	nv_mask(dev, 0x001098, 0x00000018, 0x00000018);
+
+	/* launch, and wait for completion */
+	nv_wr32(dev, 0x00130c, hwsq_kick);
+	if (!nv_wait(dev, 0x001308, 0x00000100, 0x00000000)) {
+		NV_ERROR(dev, "hwsq ucode exec timed out\n");
+		NV_ERROR(dev, "0x001308: 0x%08x\n", nv_rd32(dev, 0x001308));
+		for (i = 0; i < hwsq->len / 4; i++) {
+			NV_ERROR(dev, "0x%06x: 0x%08x\n", 0x1400 + (i * 4),
+				 nv_rd32(dev, 0x001400 + (i * 4)));
+		}
+
+		return -EIO;
+	}
+
+	return 0;
+}
+
 int
 nv50_pm_clocks_set(struct drm_device *dev, void *data)
 {
@@ -550,6 +655,28 @@ nv50_pm_clocks_set(struct drm_device *dev, void *data)
 	if (!nv_wait(dev, 0x002504, 0x00000010, 0x00000010))
 		goto error;
 
+	/* memory: it is *very* important we change this first, the ucode
+	 * we build in pre() now has hardcoded 0xc040 values, which can't
+	 * change before we execute it or the engine clocks may end up
+	 * messed up.
+	 */
+	if (info->mclk_hwsq.len) {
+		/* execute some scripts that do ??? from the vbios.. */
+		if (!bit_table(dev, 'M', &M) && M.version == 1) {
+			if (M.length >= 6)
+				nouveau_bios_init_exec(dev, ROM16(M.data[5]));
+			if (M.length >= 8)
+				nouveau_bios_init_exec(dev, ROM16(M.data[7]));
+			if (M.length >= 10)
+				nouveau_bios_init_exec(dev, ROM16(M.data[9]));
+			nouveau_bios_init_exec(dev, info->mscript);
+		}
+
+		ret = prog_mclk(dev, &info->mclk_hwsq);
+		if (ret)
+			goto resume;
+	}
+
 	/* reclock vdec/dom6 */
 	nv_mask(dev, 0x00c040, 0x00000c00, 0x00000000);
 	switch (dev_priv->chipset) {
@@ -578,50 +705,6 @@ nv50_pm_clocks_set(struct drm_device *dev, void *data)
 	nv_wr32(dev, 0x00402c, info->ncoef);
 	nv_mask(dev, 0x00c040, 0x00100033, info->emast);
 
-	/* memory */
-	if (!info->mctrl)
-		goto resume;
-
-	/* execute some scripts that do ??? from the vbios.. */
-	if (!bit_table(dev, 'M', &M) && M.version == 1) {
-		if (M.length >= 6)
-			nouveau_bios_init_exec(dev, ROM16(M.data[5]));
-		if (M.length >= 8)
-			nouveau_bios_init_exec(dev, ROM16(M.data[7]));
-		if (M.length >= 10)
-			nouveau_bios_init_exec(dev, ROM16(M.data[9]));
-		nouveau_bios_init_exec(dev, info->mscript);
-	}
-
-	/* disable display */
-	if (dev_priv->chipset >= 0x92) {
-		nv_wr32(dev, 0x611200, 0x00003300);
-		udelay(100);
-	}
-
-	/* prepare ram for reclocking */
-	nv_wr32(dev, 0x1002d4, 0x00000001); /* precharge */
-	nv_wr32(dev, 0x1002d0, 0x00000001); /* refresh */
-	nv_wr32(dev, 0x1002d0, 0x00000001); /* refresh */
-	nv_mask(dev, 0x100210, 0x80000000, 0x00000000); /* no auto-refresh */
-	nv_wr32(dev, 0x1002dc, 0x00000001); /* enable self-refresh */
-
-	/* modify mpll */
-	nv_mask(dev, 0x00c040, 0x0000c000, 0x0000c000);
-	nv_mask(dev, 0x004008, 0x01ff0200, 0x00000200 | info->mctrl);
-	nv_wr32(dev, 0x00400c, info->mcoef);
-	udelay(100);
-	nv_mask(dev, 0x004008, 0x81ff0200, info->mctrl);
-
-	/* re-enable normal operation of memory controller */
-	nv_wr32(dev, 0x1002dc, 0x00000000);
-	nv_mask(dev, 0x100210, 0x80000000, 0x80000000);
-	udelay(100);
-
-	/* re-enable display */
-	if (dev_priv->chipset >= 0x92)
-		nv_wr32(dev, 0x611200, 0x00003330);
-
 	goto resume;
 error:
 	ret = -EBUSY;