From c51d0e5c3ea5dffa4f9162b86fe4a0474180d671 Mon Sep 17 00:00:00 2001
From: Steven Webster <steven.webster@windriver.com>
Date: Mon, 22 Jun 2020 09:30:34 -0400
Subject: [PATCH] Handle N3000 FPGA reset

A recent config commit (6d639fca5d) was introduced that resets any
Intel N3000 FPGA devices on startup.  After an FPGA reset, it is
possible that the PCI addresses of the underlying FEC device may
change, in addition to the resetting of any configured drivers and
SR-IOV VF configuration on the FPGA FEC and NIC devices.

To handle the reset, it is required to do the following:

- Do not allow a user to configure an FEC device before the first
unlock (to allow the reset to occur).

- Delay the configuration of FEC and NIC devices until the reset
  has occurred.

Depends-On: https://review.opendev.org/737388
Story: 2006740
Task: 39948

Change-Id: I6d1bd988192dca868b8cc972045933ee8ac75e7c
Signed-off-by: Steven Webster <steven.webster@windriver.com>
---
 puppet-manifests/src/manifests/worker.pp      |  1 +
 .../src/modules/platform/manifests/devices.pp | 26 ++++++++++++++++---
 .../src/modules/platform/manifests/network.pp | 21 ++++++++++++---
 3 files changed, 42 insertions(+), 6 deletions(-)

diff --git a/puppet-manifests/src/manifests/worker.pp b/puppet-manifests/src/manifests/worker.pp
index 4d6359361..d0125dba5 100644
--- a/puppet-manifests/src/manifests/worker.pp
+++ b/puppet-manifests/src/manifests/worker.pp
@@ -27,6 +27,7 @@ include ::platform::remotelogging
 include ::platform::mtce
 include ::platform::sysinv
 include ::platform::devices
+include ::platform::interfaces::sriov
 include ::platform::grub
 include ::platform::collectd
 include ::platform::filesystem::compute
diff --git a/puppet-manifests/src/modules/platform/manifests/devices.pp b/puppet-manifests/src/modules/platform/manifests/devices.pp
index de53b2ce9..cfbc7c18a 100644
--- a/puppet-manifests/src/modules/platform/manifests/devices.pp
+++ b/puppet-manifests/src/modules/platform/manifests/devices.pp
@@ -43,14 +43,24 @@ class platform::devices::qat (
 define platform::devices::sriov_enable (
   $num_vfs,
   $addr,
-  $driver
+  $driver,
+  $device_id
 ) {
   if ($driver == 'igb_uio') {
     $vf_file = 'max_vfs'
   } else {
     $vf_file = 'sriov_numvfs'
   }
-  if $num_vfs {
+  if ($num_vfs > 0) {
+    if ($device_id == '0d8f') {
+      exec { "Waiting for n3000 reset before enabling device: ${title}":
+        command   => 'test -e /var/run/.sysinv_n3000_reset',
+        path      => '/usr/bin/',
+        tries     => 60,
+        try_sleep => 1,
+      }
+      -> Exec["sriov-enable-device: ${title}"]
+    }
     exec { "sriov-enable-device: ${title}":
       command   => template('platform/sriov.enable-device.erb'),
       logoutput => true,
@@ -61,9 +71,19 @@ define platform::devices::sriov_enable (
 define platform::devices::sriov_bind (
   $addr,
   $driver,
-  $num_vfs = undef
+  $num_vfs = undef,
+  $device_id = undef
 ) {
   if ($driver != undef) and ($addr != undef) {
+    if ($device_id != undef) and ($device_id == '0d8f') {
+      exec { "Waiting for n3000 reset before binding device: ${title}":
+        command   => 'test -e /var/run/.sysinv_n3000_reset',
+        path      => '/usr/bin/',
+        tries     => 60,
+        try_sleep => 1,
+      }
+      -> Exec["sriov-bind-device: ${title}"]
+    }
     ensure_resource(kmod::load, $driver)
     exec { "sriov-bind-device: ${title}":
       command   => template('platform/sriov.bind-device.erb'),
diff --git a/puppet-manifests/src/modules/platform/manifests/network.pp b/puppet-manifests/src/modules/platform/manifests/network.pp
index 17817b3b2..1470637e5 100644
--- a/puppet-manifests/src/modules/platform/manifests/network.pp
+++ b/puppet-manifests/src/modules/platform/manifests/network.pp
@@ -178,11 +178,13 @@ class platform::network::routes (
 
 define platform::interfaces::sriov_enable (
   $addr,
+  $device_id,
   $num_vfs,
+  $port_name,
   $vf_config = undef
 ) {
   $vf_file = 'sriov_numvfs'
-  if $num_vfs {
+  if ($num_vfs > 0) {
     exec { "sriov-enable-device: ${title}":
       command   => template('platform/sriov.enable-device.erb'),
       logoutput => true,
@@ -208,9 +210,24 @@ define platform::interfaces::sriov_bind (
 
 define platform::interfaces::sriov_vf_bind (
   $addr,
+  $device_id,
   $num_vfs,
+  $port_name,
   $vf_config
 ) {
+  if ($device_id == '0d58') {
+    exec { "Waiting for n3000 reset before binding device: ${title}":
+      command   => 'test -e /var/run/.sysinv_n3000_reset',
+      path      => '/usr/bin/',
+      tries     => 60,
+      try_sleep => 1,
+    }
+    -> exec { "Restarting n3000 NICs for interface: ${title}":
+      command => "ifdown ${port_name}; ifup ${port_name}",
+      path    => '/usr/sbin/',
+    }
+    -> Anchor['platform::networking']
+  }
   create_resources('platform::interfaces::sriov_bind', $vf_config, {})
 }
 
@@ -235,8 +252,6 @@ class platform::interfaces (
   $network_config = {},
 ) {
   create_resources('network_config', $network_config, {})
-
-  include ::platform::interfaces::sriov
 }