|
@@ -19,6 +19,43 @@
|
|
|
#include <asm/sn/sn_sal.h>
|
|
|
#include "xpc.h"
|
|
|
|
|
|
+/*
|
|
|
+ * Define the number of u64s required to represent all the C-brick nasids
|
|
|
+ * as a bitmap. The cross-partition kernel modules deal only with
|
|
|
+ * C-brick nasids, thus the need for bitmaps which don't account for
|
|
|
+ * odd-numbered (non C-brick) nasids.
|
|
|
+ */
|
|
|
+#define XPC_MAX_PHYSNODES_SN2 (MAX_NUMALINK_NODES / 2)
|
|
|
+#define XP_NASID_MASK_BYTES_SN2 ((XPC_MAX_PHYSNODES_SN2 + 7) / 8)
|
|
|
+#define XP_NASID_MASK_WORDS_SN2 ((XPC_MAX_PHYSNODES_SN2 + 63) / 64)
|
|
|
+
|
|
|
+/*
|
|
|
+ * Memory for XPC's amo variables is allocated by the MSPEC driver. These
|
|
|
+ * pages are located in the lowest granule. The lowest granule uses 4k pages
|
|
|
+ * for cached references and an alternate TLB handler to never provide a
|
|
|
+ * cacheable mapping for the entire region. This will prevent speculative
|
|
|
+ * reading of cached copies of our lines from being issued which will cause
|
|
|
+ * a PI FSB Protocol error to be generated by the SHUB. For XPC, we need 64
|
|
|
+ * amo variables (based on XP_MAX_NPARTITIONS_SN2) to identify the senders of
|
|
|
+ * NOTIFY IRQs, 128 amo variables (based on XP_NASID_MASK_WORDS_SN2) to identify
|
|
|
+ * the senders of ACTIVATE IRQs, 1 amo variable to identify which remote
|
|
|
+ * partitions (i.e., XPCs) consider themselves currently engaged with the
|
|
|
+ * local XPC and 1 amo variable to request partition deactivation.
|
|
|
+ */
|
|
|
+#define XPC_NOTIFY_IRQ_AMOS_SN2 0
|
|
|
+#define XPC_ACTIVATE_IRQ_AMOS_SN2 (XPC_NOTIFY_IRQ_AMOS_SN2 + \
|
|
|
+ XP_MAX_NPARTITIONS_SN2)
|
|
|
+#define XPC_ENGAGED_PARTITIONS_AMO_SN2 (XPC_ACTIVATE_IRQ_AMOS_SN2 + \
|
|
|
+ XP_NASID_MASK_WORDS_SN2)
|
|
|
+#define XPC_DEACTIVATE_REQUEST_AMO_SN2 (XPC_ENGAGED_PARTITIONS_AMO_SN2 + 1)
|
|
|
+
|
|
|
+/*
|
|
|
+ * Buffer used to store a local copy of portions of a remote partition's
|
|
|
+ * reserved page (either its header and part_nasids mask, or its vars).
|
|
|
+ */
|
|
|
+static char *xpc_remote_copy_buffer_sn2;
|
|
|
+static void *xpc_remote_copy_buffer_base_sn2;
|
|
|
+
|
|
|
static struct xpc_vars_sn2 *xpc_vars; /* >>> Add _sn2 suffix? */
|
|
|
static struct xpc_vars_part_sn2 *xpc_vars_part; /* >>> Add _sn2 suffix? */
|
|
|
|
|
@@ -176,7 +213,7 @@ xpc_send_activate_IRQ_sn2(u64 amos_page_pa, int from_nasid, int to_nasid,
|
|
|
int w_index = XPC_NASID_W_INDEX(from_nasid);
|
|
|
int b_index = XPC_NASID_B_INDEX(from_nasid);
|
|
|
struct amo *amos = (struct amo *)__va(amos_page_pa +
|
|
|
- (XPC_ACTIVATE_IRQ_AMOS *
|
|
|
+ (XPC_ACTIVATE_IRQ_AMOS_SN2 *
|
|
|
sizeof(struct amo)));
|
|
|
|
|
|
(void)xpc_send_IRQ_sn2(&amos[w_index], (1UL << b_index), to_nasid,
|
|
@@ -189,7 +226,7 @@ xpc_send_local_activate_IRQ_sn2(int from_nasid)
|
|
|
int w_index = XPC_NASID_W_INDEX(from_nasid);
|
|
|
int b_index = XPC_NASID_B_INDEX(from_nasid);
|
|
|
struct amo *amos = (struct amo *)__va(xpc_vars->amos_page_pa +
|
|
|
- (XPC_ACTIVATE_IRQ_AMOS *
|
|
|
+ (XPC_ACTIVATE_IRQ_AMOS_SN2 *
|
|
|
sizeof(struct amo)));
|
|
|
|
|
|
/* fake the sending and receipt of an activate IRQ from remote nasid */
|
|
@@ -395,7 +432,7 @@ xpc_indicate_partition_engaged_sn2(struct xpc_partition *part)
|
|
|
{
|
|
|
unsigned long irq_flags;
|
|
|
struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa +
|
|
|
- (XPC_ENGAGED_PARTITIONS_AMO *
|
|
|
+ (XPC_ENGAGED_PARTITIONS_AMO_SN2 *
|
|
|
sizeof(struct amo)));
|
|
|
|
|
|
local_irq_save(irq_flags);
|
|
@@ -422,7 +459,7 @@ xpc_indicate_partition_disengaged_sn2(struct xpc_partition *part)
|
|
|
struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
|
|
|
unsigned long irq_flags;
|
|
|
struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa +
|
|
|
- (XPC_ENGAGED_PARTITIONS_AMO *
|
|
|
+ (XPC_ENGAGED_PARTITIONS_AMO_SN2 *
|
|
|
sizeof(struct amo)));
|
|
|
|
|
|
local_irq_save(irq_flags);
|
|
@@ -455,7 +492,7 @@ xpc_indicate_partition_disengaged_sn2(struct xpc_partition *part)
|
|
|
static int
|
|
|
xpc_partition_engaged_sn2(short partid)
|
|
|
{
|
|
|
- struct amo *amo = xpc_vars->amos_page + XPC_ENGAGED_PARTITIONS_AMO;
|
|
|
+ struct amo *amo = xpc_vars->amos_page + XPC_ENGAGED_PARTITIONS_AMO_SN2;
|
|
|
|
|
|
/* our partition's amo variable ANDed with partid mask */
|
|
|
return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) &
|
|
@@ -465,7 +502,7 @@ xpc_partition_engaged_sn2(short partid)
|
|
|
static int
|
|
|
xpc_any_partition_engaged_sn2(void)
|
|
|
{
|
|
|
- struct amo *amo = xpc_vars->amos_page + XPC_ENGAGED_PARTITIONS_AMO;
|
|
|
+ struct amo *amo = xpc_vars->amos_page + XPC_ENGAGED_PARTITIONS_AMO_SN2;
|
|
|
|
|
|
/* our partition's amo variable */
|
|
|
return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) != 0;
|
|
@@ -474,7 +511,7 @@ xpc_any_partition_engaged_sn2(void)
|
|
|
static void
|
|
|
xpc_assume_partition_disengaged_sn2(short partid)
|
|
|
{
|
|
|
- struct amo *amo = xpc_vars->amos_page + XPC_ENGAGED_PARTITIONS_AMO;
|
|
|
+ struct amo *amo = xpc_vars->amos_page + XPC_ENGAGED_PARTITIONS_AMO_SN2;
|
|
|
|
|
|
/* clear bit(s) based on partid mask in our partition's amo */
|
|
|
FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
|
|
@@ -599,12 +636,12 @@ xpc_rsvd_page_init_sn2(struct xpc_rsvd_page *rp)
|
|
|
xp_max_npartitions);
|
|
|
|
|
|
/* initialize the activate IRQ related amo variables */
|
|
|
- for (i = 0; i < xp_nasid_mask_words; i++)
|
|
|
- (void)xpc_init_IRQ_amo_sn2(XPC_ACTIVATE_IRQ_AMOS + i);
|
|
|
+ for (i = 0; i < xpc_nasid_mask_words; i++)
|
|
|
+ (void)xpc_init_IRQ_amo_sn2(XPC_ACTIVATE_IRQ_AMOS_SN2 + i);
|
|
|
|
|
|
/* initialize the engaged remote partitions related amo variables */
|
|
|
- (void)xpc_init_IRQ_amo_sn2(XPC_ENGAGED_PARTITIONS_AMO);
|
|
|
- (void)xpc_init_IRQ_amo_sn2(XPC_DEACTIVATE_REQUEST_AMO);
|
|
|
+ (void)xpc_init_IRQ_amo_sn2(XPC_ENGAGED_PARTITIONS_AMO_SN2);
|
|
|
+ (void)xpc_init_IRQ_amo_sn2(XPC_DEACTIVATE_REQUEST_AMO_SN2);
|
|
|
|
|
|
return xpSuccess;
|
|
|
}
|
|
@@ -657,7 +694,7 @@ xpc_check_remote_hb_sn2(void)
|
|
|
short partid;
|
|
|
enum xp_retval ret;
|
|
|
|
|
|
- remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer;
|
|
|
+ remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2;
|
|
|
|
|
|
for (partid = 0; partid < xp_max_npartitions; partid++) {
|
|
|
|
|
@@ -749,7 +786,7 @@ xpc_request_partition_deactivation_sn2(struct xpc_partition *part)
|
|
|
struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
|
|
|
unsigned long irq_flags;
|
|
|
struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa +
|
|
|
- (XPC_DEACTIVATE_REQUEST_AMO *
|
|
|
+ (XPC_DEACTIVATE_REQUEST_AMO_SN2 *
|
|
|
sizeof(struct amo)));
|
|
|
|
|
|
local_irq_save(irq_flags);
|
|
@@ -784,7 +821,7 @@ xpc_cancel_partition_deactivation_request_sn2(struct xpc_partition *part)
|
|
|
{
|
|
|
unsigned long irq_flags;
|
|
|
struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa +
|
|
|
- (XPC_DEACTIVATE_REQUEST_AMO *
|
|
|
+ (XPC_DEACTIVATE_REQUEST_AMO_SN2 *
|
|
|
sizeof(struct amo)));
|
|
|
|
|
|
local_irq_save(irq_flags);
|
|
@@ -808,7 +845,7 @@ xpc_cancel_partition_deactivation_request_sn2(struct xpc_partition *part)
|
|
|
static int
|
|
|
xpc_partition_deactivation_requested_sn2(short partid)
|
|
|
{
|
|
|
- struct amo *amo = xpc_vars->amos_page + XPC_DEACTIVATE_REQUEST_AMO;
|
|
|
+ struct amo *amo = xpc_vars->amos_page + XPC_DEACTIVATE_REQUEST_AMO_SN2;
|
|
|
|
|
|
/* our partition's amo variable ANDed with partid mask */
|
|
|
return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) &
|
|
@@ -898,7 +935,7 @@ xpc_identify_activate_IRQ_req_sn2(int nasid)
|
|
|
|
|
|
/* pull over the reserved page structure */
|
|
|
|
|
|
- remote_rp = (struct xpc_rsvd_page *)xpc_remote_copy_buffer;
|
|
|
+ remote_rp = (struct xpc_rsvd_page *)xpc_remote_copy_buffer_sn2;
|
|
|
|
|
|
ret = xpc_get_remote_rp(nasid, NULL, remote_rp, &remote_rp_pa);
|
|
|
if (ret != xpSuccess) {
|
|
@@ -917,7 +954,7 @@ xpc_identify_activate_IRQ_req_sn2(int nasid)
|
|
|
|
|
|
/* pull over the cross partition variables */
|
|
|
|
|
|
- remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer;
|
|
|
+ remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2;
|
|
|
|
|
|
ret = xpc_get_remote_vars_sn2(remote_vars_pa, remote_vars);
|
|
|
if (ret != xpSuccess) {
|
|
@@ -996,10 +1033,10 @@ xpc_identify_activate_IRQ_sender_sn2(void)
|
|
|
int n_IRQs_detected = 0;
|
|
|
struct amo *act_amos;
|
|
|
|
|
|
- act_amos = xpc_vars->amos_page + XPC_ACTIVATE_IRQ_AMOS;
|
|
|
+ act_amos = xpc_vars->amos_page + XPC_ACTIVATE_IRQ_AMOS_SN2;
|
|
|
|
|
|
/* scan through act amo variable looking for non-zero entries */
|
|
|
- for (word = 0; word < xp_nasid_mask_words; word++) {
|
|
|
+ for (word = 0; word < xpc_nasid_mask_words; word++) {
|
|
|
|
|
|
if (xpc_exiting)
|
|
|
break;
|
|
@@ -2334,6 +2371,7 @@ int
|
|
|
xpc_init_sn2(void)
|
|
|
{
|
|
|
int ret;
|
|
|
+ size_t buf_size;
|
|
|
|
|
|
xpc_rsvd_page_init = xpc_rsvd_page_init_sn2;
|
|
|
xpc_increment_heartbeat = xpc_increment_heartbeat_sn2;
|
|
@@ -2378,6 +2416,16 @@ xpc_init_sn2(void)
|
|
|
xpc_send_msg = xpc_send_msg_sn2;
|
|
|
xpc_received_msg = xpc_received_msg_sn2;
|
|
|
|
|
|
+ buf_size = max(XPC_RP_VARS_SIZE,
|
|
|
+ XPC_RP_HEADER_SIZE + XP_NASID_MASK_BYTES_SN2);
|
|
|
+ xpc_remote_copy_buffer_sn2 = xpc_kmalloc_cacheline_aligned(buf_size,
|
|
|
+ GFP_KERNEL,
|
|
|
+ &xpc_remote_copy_buffer_base_sn2);
|
|
|
+ if (xpc_remote_copy_buffer_sn2 == NULL) {
|
|
|
+ dev_err(xpc_part, "can't get memory for remote copy buffer\n");
|
|
|
+ return -ENOMEM;
|
|
|
+ }
|
|
|
+
|
|
|
/* open up protections for IPI and [potentially] amo operations */
|
|
|
xpc_allow_IPI_ops_sn2();
|
|
|
xpc_allow_amo_ops_shub_wars_1_1_sn2();
|
|
@@ -2394,6 +2442,7 @@ xpc_init_sn2(void)
|
|
|
dev_err(xpc_part, "can't register ACTIVATE IRQ handler, "
|
|
|
"errno=%d\n", -ret);
|
|
|
xpc_disallow_IPI_ops_sn2();
|
|
|
+ kfree(xpc_remote_copy_buffer_base_sn2);
|
|
|
}
|
|
|
return ret;
|
|
|
}
|
|
@@ -2403,4 +2452,5 @@ xpc_exit_sn2(void)
|
|
|
{
|
|
|
free_irq(SGI_XPC_ACTIVATE, NULL);
|
|
|
xpc_disallow_IPI_ops_sn2();
|
|
|
+ kfree(xpc_remote_copy_buffer_base_sn2);
|
|
|
}
|